diff options
Diffstat (limited to 'js/src/jit/shared')
18 files changed, 11683 insertions, 0 deletions
diff --git a/js/src/jit/shared/Architecture-shared.h b/js/src/jit/shared/Architecture-shared.h new file mode 100644 index 0000000000..33085a6bdb --- /dev/null +++ b/js/src/jit/shared/Architecture-shared.h @@ -0,0 +1,18 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_Architecture_shared_h +#define jit_shared_Architecture_shared_h + +namespace js { +namespace jit { + +enum class RegTypeName { GPR, Float32, Float64, Vector128, Any }; + +} // namespace jit +} // namespace js + +#endif /* jit_shared_Architecture_shared_h */ diff --git a/js/src/jit/shared/Assembler-shared.cpp b/js/src/jit/shared/Assembler-shared.cpp new file mode 100644 index 0000000000..96dcf69a72 --- /dev/null +++ b/js/src/jit/shared/Assembler-shared.cpp @@ -0,0 +1,74 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/shared/Assembler-shared.h" + +#include "jit/JitSpewer.h" +#include "vm/NativeObject.h" + +namespace js::jit { + +void BaseObjectElementIndex::staticAssertions() { + NativeObject::elementsSizeMustNotOverflow(); +} + +void BaseObjectSlotIndex::staticAssertions() { + NativeObject::slotsSizeMustNotOverflow(); +} + +AssemblerShared::~AssemblerShared() { +#ifdef DEBUG + while (hasCreator()) { + popCreator(); + } +#endif +} + +#ifdef DEBUG +void AssemblerShared::pushCreator(const char* who) { + (void)creators_.append(who); + JitSpewStart(JitSpew_Codegen, "# BEGIN creators: "); + bool first = true; + for (const char* str : creators_) { + JitSpewCont(JitSpew_Codegen, "%s%s", first ? "" : "/", str); + first = false; + } + JitSpewCont(JitSpew_Codegen, "\n"); +} + +void AssemblerShared::popCreator() { + JitSpewStart(JitSpew_Codegen, "# END creators: "); + bool first = true; + for (const char* str : creators_) { + JitSpewCont(JitSpew_Codegen, "%s%s", first ? "" : "/", str); + first = false; + } + JitSpewCont(JitSpew_Codegen, "\n"); + if (creators_.empty()) { + JitSpew(JitSpew_Codegen, " "); + } + MOZ_ASSERT(!creators_.empty()); + creators_.popBack(); +} + +bool AssemblerShared::hasCreator() const { + // If you get failures of assertions of the form `MOZ_ASSERT(hasCreator())`, + // what this means is that a `MacroAssembler` (or, really, anything that + // inherits from `js::jit::AssemblerShared`) has emitted code or data from a + // place, in the SM C++ hierarchy, that is not nested within an + // `AutoCreatedBy` RAII scope. Consequently the emitted instructions/data + // won't have any owner that is identifiable in the `IONFLAGS=codegen` + // output. + // + // Fixing this is easy: work back up the crash stack and decide on a place + // to put an `AutoCreatedBy` call. A bit of grepping for `AutoCreatedBy` + // should make it obvious what to do. If in doubt, add `AutoCreatedBy` + // calls liberally; "extra" ones are harmless. + return !creators_.empty(); +} +#endif + +} // namespace js::jit diff --git a/js/src/jit/shared/Assembler-shared.h b/js/src/jit/shared/Assembler-shared.h new file mode 100644 index 0000000000..d3e66fe7a4 --- /dev/null +++ b/js/src/jit/shared/Assembler-shared.h @@ -0,0 +1,716 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_Assembler_shared_h +#define jit_shared_Assembler_shared_h + +#include "mozilla/CheckedInt.h" + +#include <limits.h> + +#include "gc/Barrier.h" +#include "jit/AtomicOp.h" +#include "jit/JitAllocPolicy.h" +#include "jit/JitCode.h" +#include "jit/JitContext.h" +#include "jit/Label.h" +#include "jit/Registers.h" +#include "jit/RegisterSets.h" +#include "js/ScalarType.h" // js::Scalar::Type +#include "vm/HelperThreads.h" +#include "wasm/WasmCodegenTypes.h" +#include "wasm/WasmConstants.h" + +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_WASM32) || \ + defined(JS_CODEGEN_RISCV64) +// Push return addresses callee-side. +# define JS_USE_LINK_REGISTER +#endif + +#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_LOONG64) || \ + defined(JS_CODEGEN_RISCV64) +// JS_CODELABEL_LINKMODE gives labels additional metadata +// describing how Bind() should patch them. +# define JS_CODELABEL_LINKMODE +#endif + +namespace js { +namespace jit { + +enum class FrameType; +enum class ExceptionResumeKind : int32_t; + +namespace Disassembler { +class HeapAccess; +} // namespace Disassembler + +static constexpr uint32_t Simd128DataSize = 4 * sizeof(int32_t); +static_assert(Simd128DataSize == 4 * sizeof(int32_t), + "SIMD data should be able to contain int32x4"); +static_assert(Simd128DataSize == 4 * sizeof(float), + "SIMD data should be able to contain float32x4"); +static_assert(Simd128DataSize == 2 * sizeof(double), + "SIMD data should be able to contain float64x2"); + +enum Scale { TimesOne = 0, TimesTwo = 1, TimesFour = 2, TimesEight = 3 }; + +static_assert(sizeof(JS::Value) == 8, + "required for TimesEight and 3 below to be correct"); +static const Scale ValueScale = TimesEight; +static const size_t ValueShift = 3; + +static inline unsigned ScaleToShift(Scale scale) { return unsigned(scale); } + +static inline bool IsShiftInScaleRange(int i) { + return i >= TimesOne && i <= TimesEight; +} + +static inline Scale ShiftToScale(int i) { + MOZ_ASSERT(IsShiftInScaleRange(i)); + return Scale(i); +} + +static inline Scale ScaleFromElemWidth(int shift) { + switch (shift) { + case 1: + return TimesOne; + case 2: + return TimesTwo; + case 4: + return TimesFour; + case 8: + return TimesEight; + } + + MOZ_CRASH("Invalid scale"); +} + +static inline Scale ScaleFromScalarType(Scalar::Type type) { + return ScaleFromElemWidth(Scalar::byteSize(type)); +} + +// Used for 32-bit immediates which do not require relocation. +struct Imm32 { + int32_t value; + + explicit Imm32(int32_t value) : value(value) {} + explicit Imm32(FrameType type) : Imm32(int32_t(type)) {} + explicit Imm32(ExceptionResumeKind kind) : Imm32(int32_t(kind)) {} + + static inline Imm32 ShiftOf(enum Scale s) { + switch (s) { + case TimesOne: + return Imm32(0); + case TimesTwo: + return Imm32(1); + case TimesFour: + return Imm32(2); + case TimesEight: + return Imm32(3); + }; + MOZ_CRASH("Invalid scale"); + } + + static inline Imm32 FactorOf(enum Scale s) { + return Imm32(1 << ShiftOf(s).value); + } +}; + +// Pointer-sized integer to be embedded as an immediate in an instruction. +struct ImmWord { + uintptr_t value; + + explicit ImmWord(uintptr_t value) : value(value) {} +}; + +// Used for 64-bit immediates which do not require relocation. +struct Imm64 { + uint64_t value; + + explicit Imm64(int64_t value) : value(value) {} + + Imm32 low() const { return Imm32(int32_t(value)); } + + Imm32 hi() const { return Imm32(int32_t(value >> 32)); } + + inline Imm32 firstHalf() const; + inline Imm32 secondHalf() const; +}; + +#ifdef DEBUG +static inline bool IsCompilingWasm() { + return GetJitContext()->isCompilingWasm(); +} +#endif + +// Pointer to be embedded as an immediate in an instruction. +struct ImmPtr { + void* value; + + struct NoCheckToken {}; + + explicit constexpr ImmPtr(std::nullptr_t) : value(nullptr) { + // Explicit constructor for nullptr. This ensures ImmPtr(0) can't be called. + // Either use ImmPtr(nullptr) or ImmWord(0). + } + + explicit ImmPtr(void* value, NoCheckToken) : value(value) { + // A special unchecked variant for contexts where we know it is safe to + // use an immptr. This is assuming the caller knows what they're doing. + } + + explicit ImmPtr(const void* value) : value(const_cast<void*>(value)) { + // To make code serialization-safe, wasm compilation should only + // compile pointer immediates using a SymbolicAddress. + MOZ_ASSERT(!IsCompilingWasm()); + } + + template <class R> + explicit ImmPtr(R (*pf)()) : value(JS_FUNC_TO_DATA_PTR(void*, pf)) { + MOZ_ASSERT(!IsCompilingWasm()); + } + + template <class R, class A1> + explicit ImmPtr(R (*pf)(A1)) : value(JS_FUNC_TO_DATA_PTR(void*, pf)) { + MOZ_ASSERT(!IsCompilingWasm()); + } + + template <class R, class A1, class A2> + explicit ImmPtr(R (*pf)(A1, A2)) : value(JS_FUNC_TO_DATA_PTR(void*, pf)) { + MOZ_ASSERT(!IsCompilingWasm()); + } + + template <class R, class A1, class A2, class A3> + explicit ImmPtr(R (*pf)(A1, A2, A3)) : value(JS_FUNC_TO_DATA_PTR(void*, pf)) { + MOZ_ASSERT(!IsCompilingWasm()); + } + + template <class R, class A1, class A2, class A3, class A4> + explicit ImmPtr(R (*pf)(A1, A2, A3, A4)) + : value(JS_FUNC_TO_DATA_PTR(void*, pf)) { + MOZ_ASSERT(!IsCompilingWasm()); + } +}; + +// The same as ImmPtr except that the intention is to patch this +// instruction. The initial value of the immediate is 'addr' and this value is +// either clobbered or used in the patching process. +struct PatchedImmPtr { + void* value; + + explicit PatchedImmPtr() : value(nullptr) {} + explicit PatchedImmPtr(const void* value) : value(const_cast<void*>(value)) {} +}; + +class AssemblerShared; +class ImmGCPtr; + +// Used for immediates which require relocation. +class ImmGCPtr { + public: + const gc::Cell* value; + + explicit ImmGCPtr(const gc::Cell* ptr) : value(ptr) { + // Nursery pointers can't be used if the main thread might be currently + // performing a minor GC. + MOZ_ASSERT_IF(ptr && !ptr->isTenured(), !CurrentThreadIsIonCompiling()); + + // wasm shouldn't be creating GC things + MOZ_ASSERT(!IsCompilingWasm()); + } + + private: + ImmGCPtr() : value(0) {} +}; + +// Pointer to trampoline code. Trampoline code is kept alive until the runtime +// is destroyed, so does not need to be traced. +struct TrampolinePtr { + uint8_t* value; + + TrampolinePtr() : value(nullptr) {} + explicit TrampolinePtr(uint8_t* value) : value(value) { MOZ_ASSERT(value); } +}; + +// Pointer to be embedded as an immediate that is loaded/stored from by an +// instruction. +struct AbsoluteAddress { + void* addr; + + explicit AbsoluteAddress(const void* addr) : addr(const_cast<void*>(addr)) { + MOZ_ASSERT(!IsCompilingWasm()); + } + + AbsoluteAddress offset(ptrdiff_t delta) { + return AbsoluteAddress(((uint8_t*)addr) + delta); + } +}; + +// The same as AbsoluteAddress except that the intention is to patch this +// instruction. The initial value of the immediate is 'addr' and this value is +// either clobbered or used in the patching process. +struct PatchedAbsoluteAddress { + void* addr; + + explicit PatchedAbsoluteAddress() : addr(nullptr) {} + explicit PatchedAbsoluteAddress(const void* addr) + : addr(const_cast<void*>(addr)) {} + explicit PatchedAbsoluteAddress(uintptr_t addr) + : addr(reinterpret_cast<void*>(addr)) {} +}; + +// Specifies an address computed in the form of a register base and a constant, +// 32-bit offset. +struct Address { + RegisterOrSP base; + int32_t offset; + + Address(Register base, int32_t offset) + : base(RegisterOrSP(base)), offset(offset) {} + +#ifdef JS_HAS_HIDDEN_SP + Address(RegisterOrSP base, int32_t offset) : base(base), offset(offset) {} +#endif + + Address() = delete; +}; + +#if JS_BITS_PER_WORD == 32 + +static inline Address LowWord(const Address& address) { + using mozilla::CheckedInt; + + CheckedInt<int32_t> offset = + CheckedInt<int32_t>(address.offset) + INT64LOW_OFFSET; + MOZ_ALWAYS_TRUE(offset.isValid()); + return Address(address.base, offset.value()); +} + +static inline Address HighWord(const Address& address) { + using mozilla::CheckedInt; + + CheckedInt<int32_t> offset = + CheckedInt<int32_t>(address.offset) + INT64HIGH_OFFSET; + MOZ_ALWAYS_TRUE(offset.isValid()); + return Address(address.base, offset.value()); +} + +#endif + +// Specifies an address computed in the form of a register base, a register +// index with a scale, and a constant, 32-bit offset. +struct BaseIndex { + RegisterOrSP base; + Register index; + Scale scale; + int32_t offset; + + BaseIndex(Register base, Register index, Scale scale, int32_t offset = 0) + : base(RegisterOrSP(base)), index(index), scale(scale), offset(offset) {} + +#ifdef JS_HAS_HIDDEN_SP + BaseIndex(RegisterOrSP base, Register index, Scale scale, int32_t offset = 0) + : base(base), index(index), scale(scale), offset(offset) {} +#endif + + BaseIndex() = delete; +}; + +#if JS_BITS_PER_WORD == 32 + +static inline BaseIndex LowWord(const BaseIndex& address) { + using mozilla::CheckedInt; + + CheckedInt<int32_t> offset = + CheckedInt<int32_t>(address.offset) + INT64LOW_OFFSET; + MOZ_ALWAYS_TRUE(offset.isValid()); + return BaseIndex(address.base, address.index, address.scale, offset.value()); +} + +static inline BaseIndex HighWord(const BaseIndex& address) { + using mozilla::CheckedInt; + + CheckedInt<int32_t> offset = + CheckedInt<int32_t>(address.offset) + INT64HIGH_OFFSET; + MOZ_ALWAYS_TRUE(offset.isValid()); + return BaseIndex(address.base, address.index, address.scale, offset.value()); +} + +#endif + +// A BaseIndex used to access Values. Note that |offset| is *not* scaled by +// sizeof(Value). Use this *only* if you're indexing into a series of Values +// that aren't object elements or object slots (for example, values on the +// stack, values in an arguments object, &c.). If you're indexing into an +// object's elements or slots, don't use this directly! Use +// BaseObject{Element,Slot}Index instead. +struct BaseValueIndex : BaseIndex { + BaseValueIndex(Register base, Register index, int32_t offset = 0) + : BaseIndex(RegisterOrSP(base), index, ValueScale, offset) {} + +#ifdef JS_HAS_HIDDEN_SP + BaseValueIndex(RegisterOrSP base, Register index, int32_t offset = 0) + : BaseIndex(base, index, ValueScale, offset) {} +#endif +}; + +// Specifies the address of an indexed Value within object elements from a +// base. The index must not already be scaled by sizeof(Value)! +struct BaseObjectElementIndex : BaseValueIndex { + BaseObjectElementIndex(Register base, Register index, int32_t offset = 0) + : BaseValueIndex(base, index, offset) {} + +#ifdef JS_HAS_HIDDEN_SP + BaseObjectElementIndex(RegisterOrSP base, Register index, int32_t offset = 0) + : BaseValueIndex(base, index, offset) {} +#endif + + static void staticAssertions(); +}; + +// Like BaseObjectElementIndex, except for object slots. +struct BaseObjectSlotIndex : BaseValueIndex { + BaseObjectSlotIndex(Register base, Register index) + : BaseValueIndex(base, index) {} + +#ifdef JS_HAS_HIDDEN_SP + BaseObjectSlotIndex(RegisterOrSP base, Register index) + : BaseValueIndex(base, index) {} +#endif + + static void staticAssertions(); +}; + +enum class RelocationKind { + // The target is immovable, so patching is only needed if the source + // buffer is relocated and the reference is relative. + HARDCODED, + + // The target is the start of a JitCode buffer, which must be traced + // during garbage collection. Relocations and patching may be needed. + JITCODE +}; + +class CodeOffset { + size_t offset_; + + static const size_t NOT_BOUND = size_t(-1); + + public: + explicit CodeOffset(size_t offset) : offset_(offset) {} + CodeOffset() : offset_(NOT_BOUND) {} + + size_t offset() const { + MOZ_ASSERT(bound()); + return offset_; + } + + void bind(size_t offset) { + MOZ_ASSERT(!bound()); + offset_ = offset; + MOZ_ASSERT(bound()); + } + bool bound() const { return offset_ != NOT_BOUND; } + + void offsetBy(size_t delta) { + MOZ_ASSERT(bound()); + MOZ_ASSERT(offset_ + delta >= offset_, "no overflow"); + offset_ += delta; + } +}; + +// A code label contains an absolute reference to a point in the code. Thus, it +// cannot be patched until after linking. +// When the source label is resolved into a memory address, this address is +// patched into the destination address. +// Some need to distinguish between multiple ways of patching that address. +// See JS_CODELABEL_LINKMODE. +class CodeLabel { + // The destination position, where the absolute reference should get + // patched into. + CodeOffset patchAt_; + + // The source label (relative) in the code to where the destination should + // get patched to. + CodeOffset target_; + +#ifdef JS_CODELABEL_LINKMODE + public: + enum LinkMode { Uninitialized = 0, RawPointer, MoveImmediate, JumpImmediate }; + + private: + LinkMode linkMode_ = Uninitialized; +#endif + + public: + CodeLabel() = default; + explicit CodeLabel(const CodeOffset& patchAt) : patchAt_(patchAt) {} + CodeLabel(const CodeOffset& patchAt, const CodeOffset& target) + : patchAt_(patchAt), target_(target) {} + CodeOffset* patchAt() { return &patchAt_; } + CodeOffset* target() { return &target_; } + CodeOffset patchAt() const { return patchAt_; } + CodeOffset target() const { return target_; } +#ifdef JS_CODELABEL_LINKMODE + LinkMode linkMode() const { return linkMode_; } + void setLinkMode(LinkMode value) { linkMode_ = value; } +#endif +}; + +typedef Vector<CodeLabel, 0, SystemAllocPolicy> CodeLabelVector; + +class CodeLocationLabel { + uint8_t* raw_ = nullptr; + + public: + CodeLocationLabel(JitCode* code, CodeOffset base) { + MOZ_ASSERT(base.offset() < code->instructionsSize()); + raw_ = code->raw() + base.offset(); + } + explicit CodeLocationLabel(JitCode* code) { raw_ = code->raw(); } + explicit CodeLocationLabel(uint8_t* raw) { + MOZ_ASSERT(raw); + raw_ = raw; + } + + ptrdiff_t operator-(const CodeLocationLabel& other) const { + return raw_ - other.raw_; + } + + uint8_t* raw() const { return raw_; } +}; + +} // namespace jit + +namespace wasm { + +// Represents an instruction to be patched and the intended pointee. These +// links are accumulated in the MacroAssembler, but patching is done outside +// the MacroAssembler (in Module::staticallyLink). + +struct SymbolicAccess { + SymbolicAccess(jit::CodeOffset patchAt, SymbolicAddress target) + : patchAt(patchAt), target(target) {} + + jit::CodeOffset patchAt; + SymbolicAddress target; +}; + +typedef Vector<SymbolicAccess, 0, SystemAllocPolicy> SymbolicAccessVector; + +// Describes a single wasm or asm.js memory access for the purpose of generating +// code and metadata. + +class MemoryAccessDesc { + uint64_t offset64_; + uint32_t align_; + Scalar::Type type_; + jit::Synchronization sync_; + wasm::BytecodeOffset trapOffset_; + wasm::SimdOp widenOp_; + enum { Plain, ZeroExtend, Splat, Widen } loadOp_; + + public: + explicit MemoryAccessDesc( + Scalar::Type type, uint32_t align, uint64_t offset, + BytecodeOffset trapOffset, + const jit::Synchronization& sync = jit::Synchronization::None()) + : offset64_(offset), + align_(align), + type_(type), + sync_(sync), + trapOffset_(trapOffset), + widenOp_(wasm::SimdOp::Limit), + loadOp_(Plain) { + MOZ_ASSERT(mozilla::IsPowerOfTwo(align)); + } + + // The offset is a 64-bit value because of memory64. Almost always, it will + // fit in 32 bits, and hence offset() checks that it will, this method is used + // almost everywhere in the engine. The compiler front-ends must use + // offset64() to bypass the check performed by offset(), and must resolve + // offsets that don't fit in 32 bits early in the compilation pipeline so that + // no large offsets are observed later. + uint32_t offset() const { + MOZ_ASSERT(offset64_ <= UINT32_MAX); + return uint32_t(offset64_); + } + uint64_t offset64() const { return offset64_; } + + // The offset can be cleared without worrying about its magnitude. + void clearOffset() { offset64_ = 0; } + + // The offset can be set (after compile-time evaluation) but only to values + // that fit in 32 bits. + void setOffset32(uint32_t offset) { offset64_ = offset; } + + uint32_t align() const { return align_; } + Scalar::Type type() const { return type_; } + unsigned byteSize() const { return Scalar::byteSize(type()); } + const jit::Synchronization& sync() const { return sync_; } + BytecodeOffset trapOffset() const { return trapOffset_; } + wasm::SimdOp widenSimdOp() const { + MOZ_ASSERT(isWidenSimd128Load()); + return widenOp_; + } + bool isAtomic() const { return !sync_.isNone(); } + bool isZeroExtendSimd128Load() const { return loadOp_ == ZeroExtend; } + bool isSplatSimd128Load() const { return loadOp_ == Splat; } + bool isWidenSimd128Load() const { return loadOp_ == Widen; } + + void setZeroExtendSimd128Load() { + MOZ_ASSERT(type() == Scalar::Float32 || type() == Scalar::Float64); + MOZ_ASSERT(!isAtomic()); + MOZ_ASSERT(loadOp_ == Plain); + loadOp_ = ZeroExtend; + } + + void setSplatSimd128Load() { + MOZ_ASSERT(type() == Scalar::Uint8 || type() == Scalar::Uint16 || + type() == Scalar::Float32 || type() == Scalar::Float64); + MOZ_ASSERT(!isAtomic()); + MOZ_ASSERT(loadOp_ == Plain); + loadOp_ = Splat; + } + + void setWidenSimd128Load(wasm::SimdOp op) { + MOZ_ASSERT(type() == Scalar::Float64); + MOZ_ASSERT(!isAtomic()); + MOZ_ASSERT(loadOp_ == Plain); + widenOp_ = op; + loadOp_ = Widen; + } +}; + +} // namespace wasm + +namespace jit { + +// The base class of all Assemblers for all archs. +class AssemblerShared { + wasm::CallSiteVector callSites_; + wasm::CallSiteTargetVector callSiteTargets_; + wasm::TrapSiteVectorArray trapSites_; + wasm::SymbolicAccessVector symbolicAccesses_; + wasm::TryNoteVector tryNotes_; +#ifdef DEBUG + // To facilitate figuring out which part of SM created each instruction as + // shown by IONFLAGS=codegen, this maintains a stack of (notionally) + // code-creating routines, which is printed in the log output every time an + // entry is pushed or popped. Do not push/pop entries directly; instead use + // `class AutoCreatedBy`. + mozilla::Vector<const char*> creators_; +#endif + + protected: + CodeLabelVector codeLabels_; + + bool enoughMemory_; + bool embedsNurseryPointers_; + + public: + AssemblerShared() : enoughMemory_(true), embedsNurseryPointers_(false) {} + + ~AssemblerShared(); + +#ifdef DEBUG + // Do not use these directly; instead use `class AutoCreatedBy`. + void pushCreator(const char*); + void popCreator(); + // See comment on the implementation of `hasCreator` for guidance on what to + // do if you get failures of the assertion `MOZ_ASSERT(hasCreator())`, + bool hasCreator() const; +#endif + + void propagateOOM(bool success) { enoughMemory_ &= success; } + + void setOOM() { enoughMemory_ = false; } + + bool oom() const { return !enoughMemory_; } + + bool embedsNurseryPointers() const { return embedsNurseryPointers_; } + + void addCodeLabel(CodeLabel label) { + propagateOOM(codeLabels_.append(label)); + } + size_t numCodeLabels() const { return codeLabels_.length(); } + CodeLabel codeLabel(size_t i) { return codeLabels_[i]; } + CodeLabelVector& codeLabels() { return codeLabels_; } + + // WebAssembly metadata emitted by masm operations accumulated on the + // MacroAssembler, and swapped into a wasm::CompiledCode after finish(). + + template <typename... Args> + void append(const wasm::CallSiteDesc& desc, CodeOffset retAddr, + Args&&... args) { + enoughMemory_ &= callSites_.emplaceBack(desc, retAddr.offset()); + enoughMemory_ &= callSiteTargets_.emplaceBack(std::forward<Args>(args)...); + } + void append(wasm::Trap trap, wasm::TrapSite site) { + enoughMemory_ &= trapSites_[trap].append(site); + } + void append(const wasm::MemoryAccessDesc& access, uint32_t pcOffset) { + appendOutOfBoundsTrap(access.trapOffset(), pcOffset); + } + void appendOutOfBoundsTrap(wasm::BytecodeOffset trapOffset, + uint32_t pcOffset) { + append(wasm::Trap::OutOfBounds, wasm::TrapSite(pcOffset, trapOffset)); + } + void append(wasm::SymbolicAccess access) { + enoughMemory_ &= symbolicAccesses_.append(access); + } + // This one returns an index as the try note so that it can be looked up + // later to add the end point and stack position of the try block. + [[nodiscard]] bool append(wasm::TryNote tryNote, size_t* tryNoteIndex) { + if (!tryNotes_.append(tryNote)) { + enoughMemory_ = false; + return false; + } + *tryNoteIndex = tryNotes_.length() - 1; + return true; + } + + wasm::CallSiteVector& callSites() { return callSites_; } + wasm::CallSiteTargetVector& callSiteTargets() { return callSiteTargets_; } + wasm::TrapSiteVectorArray& trapSites() { return trapSites_; } + wasm::SymbolicAccessVector& symbolicAccesses() { return symbolicAccesses_; } + wasm::TryNoteVector& tryNotes() { return tryNotes_; } +}; + +// AutoCreatedBy pushes and later pops a who-created-these-insns? tag into the +// JitSpew_Codegen output. These could be created fairly frequently, so a +// dummy inlineable-out version is provided for non-debug builds. The tag +// text can be completely arbitrary -- it serves only to help readers of the +// output text to relate instructions back to the part(s) of SM that created +// them. +#ifdef DEBUG +class MOZ_RAII AutoCreatedBy { + private: + AssemblerShared& ash_; + + public: + AutoCreatedBy(AssemblerShared& ash, const char* who) : ash_(ash) { + ash_.pushCreator(who); + } + ~AutoCreatedBy() { ash_.popCreator(); } +}; +#else +class MOZ_RAII AutoCreatedBy { + public: + inline AutoCreatedBy(AssemblerShared& ash, const char* who) {} + // A user-defined constructor is necessary to stop some compilers from + // complaining about unused variables. + inline ~AutoCreatedBy() {} +}; +#endif + +} // namespace jit +} // namespace js + +#endif /* jit_shared_Assembler_shared_h */ diff --git a/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h b/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h new file mode 100644 index 0000000000..d4bf3430ff --- /dev/null +++ b/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h @@ -0,0 +1,453 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* For documentation, see jit/AtomicOperations.h, both the comment block at the + * beginning and the #ifdef nest near the end. + * + * This is a common file for tier-3 platforms (including simulators for our + * tier-1 platforms) that are not providing hardware-specific implementations of + * the atomic operations. Please keep it reasonably platform-independent by + * adding #ifdefs at the beginning as much as possible, not throughout the file. + * + * + * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + * !!!! NOTE !!!! + * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + * + * The implementations in this file are NOT SAFE and cannot be safe even in + * principle because they rely on C++ undefined behavior. However, they are + * frequently good enough for tier-3 platforms. + */ + +#ifndef jit_shared_AtomicOperations_feeling_lucky_gcc_h +#define jit_shared_AtomicOperations_feeling_lucky_gcc_h + +#include "mozilla/Assertions.h" +#include "mozilla/Types.h" + +// Explicitly exclude tier-1 platforms. + +#if (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ + defined(_M_IX86) || (defined(__arm__) && __ARM_ARCH >= 7) || \ + defined(__aarch64__)) +# error "Do not use on a tier-1 platform where inline assembly is available" +#endif + +#if !(defined(__clang__) || defined(__GNUC__)) +# error "This file only for gcc/Clang" +#endif + +// 64-bit atomics are not required by the JS spec, and you can compile +// SpiderMonkey without them. 64-bit atomics are required for BigInt +// support. +// +// 64-bit lock-free atomics are required for WebAssembly, but gating in the +// WebAssembly subsystem ensures that no WebAssembly-supporting platforms need +// code in this file. + +#if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM) || \ + defined(JS_SIMULATOR_MIPS64) || defined(JS_SIMULATOR_LOONG64) +// On some x86 (32-bit) systems this will not work because the compiler does not +// open-code 64-bit atomics. If so, try linking with -latomic. If that doesn't +// work, you're mostly on your own. +# define HAS_64BIT_ATOMICS +# define HAS_64BIT_LOCKFREE +#endif + +#if defined(__arm__) +# define HAS_64BIT_ATOMICS +#endif + +#if defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ + defined(__PPC64LE__) +# define HAS_64BIT_ATOMICS +# define HAS_64BIT_LOCKFREE +#endif + +#if defined(__riscv) && __riscv_xlen == 64 +# define HAS_64BIT_ATOMICS +# define HAS_64BIT_LOCKFREE +#endif + +#if defined(__loongarch64) +# define HAS_64BIT_ATOMICS +# define HAS_64BIT_LOCKFREE +#endif + +#ifdef __sparc__ +# ifdef __LP64__ +# define HAS_64BIT_ATOMICS +# define HAS_64BIT_LOCKFREE +# endif +#endif + +#ifdef JS_CODEGEN_NONE +# ifdef JS_64BIT +# define HAS_64BIT_ATOMICS +# define HAS_64BIT_LOCKFREE +# endif +#endif + +// The default implementation tactic for gcc/clang is to use the newer __atomic +// intrinsics added for use in C++11 <atomic>. Where that isn't available, we +// use GCC's older __sync functions instead. +// +// ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS is kept as a backward compatible +// option for older compilers: enable this to use GCC's old __sync functions +// instead of the newer __atomic functions. This will be required for GCC 4.6.x +// and earlier, and probably for Clang 3.1, should we need to use those +// versions. Firefox no longer supports compilers that old. + +// #define ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + +// Sanity check. + +#if defined(HAS_64BIT_LOCKFREE) && !defined(HAS_64BIT_ATOMICS) +# error "This combination of features is senseless, please fix" +#endif + +// Try to avoid platform #ifdefs below this point. + +// When compiling with Clang on 32-bit linux it will be necessary to link with +// -latomic to get the proper 64-bit intrinsics. + +inline bool js::jit::AtomicOperations::hasAtomic8() { +#if defined(HAS_64BIT_ATOMICS) + return true; +#else + return false; +#endif +} + +inline bool js::jit::AtomicOperations::isLockfree8() { +#if defined(HAS_64BIT_LOCKFREE) + return true; +#else + return false; +#endif +} + +inline void js::jit::AtomicOperations::fenceSeqCst() { +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + __sync_synchronize(); +#else + __atomic_thread_fence(__ATOMIC_SEQ_CST); +#endif +} + +template <typename T> +inline T js::jit::AtomicOperations::loadSeqCst(T* addr) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + __sync_synchronize(); + T v = *addr; + __sync_synchronize(); +#else + T v; + __atomic_load(addr, &v, __ATOMIC_SEQ_CST); +#endif + return v; +} + +#ifndef HAS_64BIT_ATOMICS +namespace js { +namespace jit { + +template <> +inline int64_t AtomicOperations::loadSeqCst(int64_t* addr) { + MOZ_CRASH("No 64-bit atomics"); +} + +template <> +inline uint64_t AtomicOperations::loadSeqCst(uint64_t* addr) { + MOZ_CRASH("No 64-bit atomics"); +} + +} // namespace jit +} // namespace js +#endif + +template <typename T> +inline void js::jit::AtomicOperations::storeSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + __sync_synchronize(); + *addr = val; + __sync_synchronize(); +#else + __atomic_store(addr, &val, __ATOMIC_SEQ_CST); +#endif +} + +#ifndef HAS_64BIT_ATOMICS +namespace js { +namespace jit { + +template <> +inline void AtomicOperations::storeSeqCst(int64_t* addr, int64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +template <> +inline void AtomicOperations::storeSeqCst(uint64_t* addr, uint64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +} // namespace jit +} // namespace js +#endif + +template <typename T> +inline T js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + T v; + __sync_synchronize(); + do { + v = *addr; + } while (__sync_val_compare_and_swap(addr, v, val) != v); + return v; +#else + T v; + __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST); + return v; +#endif +} + +#ifndef HAS_64BIT_ATOMICS +namespace js { +namespace jit { + +template <> +inline int64_t AtomicOperations::exchangeSeqCst(int64_t* addr, int64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +template <> +inline uint64_t AtomicOperations::exchangeSeqCst(uint64_t* addr, uint64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +} // namespace jit +} // namespace js +#endif + +template <typename T> +inline T js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval, + T newval) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + return __sync_val_compare_and_swap(addr, oldval, newval); +#else + __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST, + __ATOMIC_SEQ_CST); + return oldval; +#endif +} + +#ifndef HAS_64BIT_ATOMICS +namespace js { +namespace jit { + +template <> +inline int64_t AtomicOperations::compareExchangeSeqCst(int64_t* addr, + int64_t oldval, + int64_t newval) { + MOZ_CRASH("No 64-bit atomics"); +} + +template <> +inline uint64_t AtomicOperations::compareExchangeSeqCst(uint64_t* addr, + uint64_t oldval, + uint64_t newval) { + MOZ_CRASH("No 64-bit atomics"); +} + +} // namespace jit +} // namespace js +#endif + +template <typename T> +inline T js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + return __sync_fetch_and_add(addr, val); +#else + return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST); +#endif +} + +#ifndef HAS_64BIT_ATOMICS +namespace js { +namespace jit { + +template <> +inline int64_t AtomicOperations::fetchAddSeqCst(int64_t* addr, int64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +template <> +inline uint64_t AtomicOperations::fetchAddSeqCst(uint64_t* addr, uint64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +} // namespace jit +} // namespace js +#endif + +template <typename T> +inline T js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + return __sync_fetch_and_sub(addr, val); +#else + return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST); +#endif +} + +#ifndef HAS_64BIT_ATOMICS +namespace js { +namespace jit { + +template <> +inline int64_t AtomicOperations::fetchSubSeqCst(int64_t* addr, int64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +template <> +inline uint64_t AtomicOperations::fetchSubSeqCst(uint64_t* addr, uint64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +} // namespace jit +} // namespace js +#endif + +template <typename T> +inline T js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + return __sync_fetch_and_and(addr, val); +#else + return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST); +#endif +} + +#ifndef HAS_64BIT_ATOMICS +namespace js { +namespace jit { + +template <> +inline int64_t AtomicOperations::fetchAndSeqCst(int64_t* addr, int64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +template <> +inline uint64_t AtomicOperations::fetchAndSeqCst(uint64_t* addr, uint64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +} // namespace jit +} // namespace js +#endif + +template <typename T> +inline T js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + return __sync_fetch_and_or(addr, val); +#else + return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST); +#endif +} + +#ifndef HAS_64BIT_ATOMICS +namespace js { +namespace jit { + +template <> +inline int64_t AtomicOperations::fetchOrSeqCst(int64_t* addr, int64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +template <> +inline uint64_t AtomicOperations::fetchOrSeqCst(uint64_t* addr, uint64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +} // namespace jit +} // namespace js +#endif + +template <typename T> +inline T js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); +#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS + return __sync_fetch_and_xor(addr, val); +#else + return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST); +#endif +} + +#ifndef HAS_64BIT_ATOMICS +namespace js { +namespace jit { + +template <> +inline int64_t AtomicOperations::fetchXorSeqCst(int64_t* addr, int64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +template <> +inline uint64_t AtomicOperations::fetchXorSeqCst(uint64_t* addr, uint64_t val) { + MOZ_CRASH("No 64-bit atomics"); +} + +} // namespace jit +} // namespace js +#endif + +template <typename T> +inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); + // This is actually roughly right even on 32-bit platforms since in that + // case, double, int64, and uint64 loads need not be access-atomic. + // + // We could use __atomic_load, but it would be needlessly expensive on + // 32-bit platforms that could support it and just plain wrong on others. + return *addr; +} + +template <typename T> +inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) { + static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only"); + // This is actually roughly right even on 32-bit platforms since in that + // case, double, int64, and uint64 loads need not be access-atomic. + // + // We could use __atomic_store, but it would be needlessly expensive on + // 32-bit platforms that could support it and just plain wrong on others. + *addr = val; +} + +inline void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest, + const void* src, + size_t nbytes) { + MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes)); + MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes)); + ::memcpy(dest, src, nbytes); +} + +inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest, + const void* src, + size_t nbytes) { + ::memmove(dest, src, nbytes); +} + +#undef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS +#undef HAS_64BIT_ATOMICS +#undef HAS_64BIT_LOCKFREE + +#endif // jit_shared_AtomicOperations_feeling_lucky_gcc_h diff --git a/js/src/jit/shared/AtomicOperations-feeling-lucky.h b/js/src/jit/shared/AtomicOperations-feeling-lucky.h new file mode 100644 index 0000000000..4aa7883fd4 --- /dev/null +++ b/js/src/jit/shared/AtomicOperations-feeling-lucky.h @@ -0,0 +1,16 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_AtomicOperations_feeling_lucky_h +#define jit_shared_AtomicOperations_feeling_lucky_h + +#if defined(__clang__) || defined(__GNUC__) +# include "jit/shared/AtomicOperations-feeling-lucky-gcc.h" +#else +# error "No AtomicOperations support for this platform+compiler combination" +#endif + +#endif // jit_shared_AtomicOperations_feeling_lucky_h diff --git a/js/src/jit/shared/AtomicOperations-shared-jit.cpp b/js/src/jit/shared/AtomicOperations-shared-jit.cpp new file mode 100644 index 0000000000..df7c049dfa --- /dev/null +++ b/js/src/jit/shared/AtomicOperations-shared-jit.cpp @@ -0,0 +1,180 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/AtomicOperations.h" + +#if defined(__arm__) +# include "jit/arm/Architecture-arm.h" +#endif + +#ifdef JS_HAVE_GENERATED_ATOMIC_OPS + +# include <atomic> + +# include "js/GCAPI.h" + +using namespace js; +using namespace js::jit; + +// A "block" is a sequence of bytes that is a reasonable quantum to copy to +// amortize call overhead when implementing memcpy and memmove. A block will +// not fit in registers on all platforms and copying it without using +// intermediate memory will therefore be sensitive to overlap. +// +// A "word" is an item that we can copy using only register intermediate storage +// on all platforms; words can be individually copied without worrying about +// overlap. +// +// Blocks and words can be aligned or unaligned; specific (generated) copying +// functions handle this in platform-specific ways. + +static constexpr size_t WORDSIZE = sizeof(uintptr_t); +static constexpr size_t BLOCKSIZE = 8 * WORDSIZE; // Must be a power of 2 + +static_assert(BLOCKSIZE % WORDSIZE == 0, + "A block is an integral number of words"); + +// Constants must match the ones in GenerateAtomicOperations.py +static_assert(JS_GENERATED_ATOMICS_BLOCKSIZE == BLOCKSIZE); +static_assert(JS_GENERATED_ATOMICS_WORDSIZE == WORDSIZE); + +static constexpr size_t WORDMASK = WORDSIZE - 1; +static constexpr size_t BLOCKMASK = BLOCKSIZE - 1; + +namespace js { +namespace jit { + +static bool UnalignedAccessesAreOK() { +# ifdef DEBUG + const char* flag = getenv("JS_NO_UNALIGNED_MEMCPY"); + if (flag && *flag == '1') return false; +# endif +# if defined(__x86_64__) || defined(__i386__) + return true; +# elif defined(__arm__) + return !HasAlignmentFault(); +# elif defined(__aarch64__) + // This is not necessarily true but it's the best guess right now. + return true; +# else +# error "Unsupported platform" +# endif +} + +# ifndef JS_64BIT +void AtomicCompilerFence() { + std::atomic_signal_fence(std::memory_order_acq_rel); +} +# endif + +void AtomicMemcpyDownUnsynchronized(uint8_t* dest, const uint8_t* src, + size_t nbytes) { + JS::AutoSuppressGCAnalysis nogc; + + const uint8_t* lim = src + nbytes; + + // Set up bulk copying. The cases are ordered the way they are on the + // assumption that if we can achieve aligned copies even with a little + // preprocessing then that is better than unaligned copying on a platform + // that supports it. + + if (nbytes >= WORDSIZE) { + void (*copyBlock)(uint8_t* dest, const uint8_t* src); + void (*copyWord)(uint8_t* dest, const uint8_t* src); + + if (((uintptr_t(dest) ^ uintptr_t(src)) & WORDMASK) == 0) { + const uint8_t* cutoff = (const uint8_t*)RoundUp(uintptr_t(src), WORDSIZE); + MOZ_ASSERT(cutoff <= lim); // because nbytes >= WORDSIZE + while (src < cutoff) { + AtomicCopyByteUnsynchronized(dest++, src++); + } + copyBlock = AtomicCopyBlockDownUnsynchronized; + copyWord = AtomicCopyWordUnsynchronized; + } else if (UnalignedAccessesAreOK()) { + copyBlock = AtomicCopyBlockDownUnsynchronized; + copyWord = AtomicCopyWordUnsynchronized; + } else { + copyBlock = AtomicCopyUnalignedBlockDownUnsynchronized; + copyWord = AtomicCopyUnalignedWordDownUnsynchronized; + } + + // Bulk copy, first larger blocks and then individual words. + + const uint8_t* blocklim = src + ((lim - src) & ~BLOCKMASK); + while (src < blocklim) { + copyBlock(dest, src); + dest += BLOCKSIZE; + src += BLOCKSIZE; + } + + const uint8_t* wordlim = src + ((lim - src) & ~WORDMASK); + while (src < wordlim) { + copyWord(dest, src); + dest += WORDSIZE; + src += WORDSIZE; + } + } + + // Byte copy any remaining tail. + + while (src < lim) { + AtomicCopyByteUnsynchronized(dest++, src++); + } +} + +void AtomicMemcpyUpUnsynchronized(uint8_t* dest, const uint8_t* src, + size_t nbytes) { + JS::AutoSuppressGCAnalysis nogc; + + const uint8_t* lim = src; + + src += nbytes; + dest += nbytes; + + if (nbytes >= WORDSIZE) { + void (*copyBlock)(uint8_t* dest, const uint8_t* src); + void (*copyWord)(uint8_t* dest, const uint8_t* src); + + if (((uintptr_t(dest) ^ uintptr_t(src)) & WORDMASK) == 0) { + const uint8_t* cutoff = (const uint8_t*)(uintptr_t(src) & ~WORDMASK); + MOZ_ASSERT(cutoff >= lim); // Because nbytes >= WORDSIZE + while (src > cutoff) { + AtomicCopyByteUnsynchronized(--dest, --src); + } + copyBlock = AtomicCopyBlockUpUnsynchronized; + copyWord = AtomicCopyWordUnsynchronized; + } else if (UnalignedAccessesAreOK()) { + copyBlock = AtomicCopyBlockUpUnsynchronized; + copyWord = AtomicCopyWordUnsynchronized; + } else { + copyBlock = AtomicCopyUnalignedBlockUpUnsynchronized; + copyWord = AtomicCopyUnalignedWordUpUnsynchronized; + } + + const uint8_t* blocklim = src - ((src - lim) & ~BLOCKMASK); + while (src > blocklim) { + dest -= BLOCKSIZE; + src -= BLOCKSIZE; + copyBlock(dest, src); + } + + const uint8_t* wordlim = src - ((src - lim) & ~WORDMASK); + while (src > wordlim) { + dest -= WORDSIZE; + src -= WORDSIZE; + copyWord(dest, src); + } + } + + while (src > lim) { + AtomicCopyByteUnsynchronized(--dest, --src); + } +} + +} // namespace jit +} // namespace js + +#endif // JS_HAVE_GENERATED_ATOMIC_OPS diff --git a/js/src/jit/shared/AtomicOperations-shared-jit.h b/js/src/jit/shared/AtomicOperations-shared-jit.h new file mode 100644 index 0000000000..ca66a6f9b9 --- /dev/null +++ b/js/src/jit/shared/AtomicOperations-shared-jit.h @@ -0,0 +1,490 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* For overall documentation, see jit/AtomicOperations.h. + * + * NOTE CAREFULLY: This file is only applicable when we have configured a JIT + * and the JIT is for the same architecture that we're compiling the shell for. + * Simulators must use a different mechanism. + * + * See comments before the include nest near the end of jit/AtomicOperations.h + * if you didn't understand that. + */ + +#ifndef jit_shared_AtomicOperations_shared_jit_h +#define jit_shared_AtomicOperations_shared_jit_h + +#include "mozilla/Assertions.h" + +#include <stddef.h> +#include <stdint.h> + +#include "jit/AtomicOperationsGenerated.h" +#include "vm/Uint8Clamped.h" + +namespace js { +namespace jit { + +#ifndef JS_64BIT +// `AtomicCompilerFence` erects a reordering boundary for operations on the +// current thread. We use it to prevent the compiler from reordering loads and +// stores inside larger primitives that are synthesized from cmpxchg. +extern void AtomicCompilerFence(); +#endif + +// `...MemcpyDown` moves bytes toward lower addresses in memory: dest <= src. +// `...MemcpyUp` moves bytes toward higher addresses in memory: dest >= src. +extern void AtomicMemcpyDownUnsynchronized(uint8_t* dest, const uint8_t* src, + size_t nbytes); +extern void AtomicMemcpyUpUnsynchronized(uint8_t* dest, const uint8_t* src, + size_t nbytes); + +} // namespace jit +} // namespace js + +inline bool js::jit::AtomicOperations::hasAtomic8() { return true; } + +inline bool js::jit::AtomicOperations::isLockfree8() { return true; } + +inline void js::jit::AtomicOperations::fenceSeqCst() { AtomicFenceSeqCst(); } + +#define JIT_LOADOP(T, U, loadop) \ + template <> \ + inline T AtomicOperations::loadSeqCst(T* addr) { \ + return (T)loadop((U*)addr); \ + } + +#ifndef JS_64BIT +# define JIT_LOADOP_CAS(T) \ + template <> \ + inline T AtomicOperations::loadSeqCst(T* addr) { \ + AtomicCompilerFence(); \ + return (T)AtomicCmpXchg64SeqCst((uint64_t*)addr, 0, 0); \ + } +#endif // !JS_64BIT + +namespace js { +namespace jit { + +JIT_LOADOP(int8_t, uint8_t, AtomicLoad8SeqCst) +JIT_LOADOP(uint8_t, uint8_t, AtomicLoad8SeqCst) +JIT_LOADOP(int16_t, uint16_t, AtomicLoad16SeqCst) +JIT_LOADOP(uint16_t, uint16_t, AtomicLoad16SeqCst) +JIT_LOADOP(int32_t, uint32_t, AtomicLoad32SeqCst) +JIT_LOADOP(uint32_t, uint32_t, AtomicLoad32SeqCst) + +#ifdef JIT_LOADOP_CAS +JIT_LOADOP_CAS(int64_t) +JIT_LOADOP_CAS(uint64_t) +#else +JIT_LOADOP(int64_t, uint64_t, AtomicLoad64SeqCst) +JIT_LOADOP(uint64_t, uint64_t, AtomicLoad64SeqCst) +#endif + +} // namespace jit +} // namespace js + +#undef JIT_LOADOP +#undef JIT_LOADOP_CAS + +#define JIT_STOREOP(T, U, storeop) \ + template <> \ + inline void AtomicOperations::storeSeqCst(T* addr, T val) { \ + storeop((U*)addr, val); \ + } + +#ifndef JS_64BIT +# define JIT_STOREOP_CAS(T) \ + template <> \ + inline void AtomicOperations::storeSeqCst(T* addr, T val) { \ + AtomicCompilerFence(); \ + T oldval = *addr; /* good initial approximation */ \ + for (;;) { \ + T nextval = (T)AtomicCmpXchg64SeqCst((uint64_t*)addr, \ + (uint64_t)oldval, (uint64_t)val); \ + if (nextval == oldval) { \ + break; \ + } \ + oldval = nextval; \ + } \ + AtomicCompilerFence(); \ + } +#endif // !JS_64BIT + +namespace js { +namespace jit { + +JIT_STOREOP(int8_t, uint8_t, AtomicStore8SeqCst) +JIT_STOREOP(uint8_t, uint8_t, AtomicStore8SeqCst) +JIT_STOREOP(int16_t, uint16_t, AtomicStore16SeqCst) +JIT_STOREOP(uint16_t, uint16_t, AtomicStore16SeqCst) +JIT_STOREOP(int32_t, uint32_t, AtomicStore32SeqCst) +JIT_STOREOP(uint32_t, uint32_t, AtomicStore32SeqCst) + +#ifdef JIT_STOREOP_CAS +JIT_STOREOP_CAS(int64_t) +JIT_STOREOP_CAS(uint64_t) +#else +JIT_STOREOP(int64_t, uint64_t, AtomicStore64SeqCst) +JIT_STOREOP(uint64_t, uint64_t, AtomicStore64SeqCst) +#endif + +} // namespace jit +} // namespace js + +#undef JIT_STOREOP +#undef JIT_STOREOP_CAS + +#define JIT_EXCHANGEOP(T, U, xchgop) \ + template <> \ + inline T AtomicOperations::exchangeSeqCst(T* addr, T val) { \ + return (T)xchgop((U*)addr, (U)val); \ + } + +#ifndef JS_64BIT +# define JIT_EXCHANGEOP_CAS(T) \ + template <> \ + inline T AtomicOperations::exchangeSeqCst(T* addr, T val) { \ + AtomicCompilerFence(); \ + T oldval = *addr; \ + for (;;) { \ + T nextval = (T)AtomicCmpXchg64SeqCst((uint64_t*)addr, \ + (uint64_t)oldval, (uint64_t)val); \ + if (nextval == oldval) { \ + break; \ + } \ + oldval = nextval; \ + } \ + AtomicCompilerFence(); \ + return oldval; \ + } +#endif // !JS_64BIT + +namespace js { +namespace jit { + +JIT_EXCHANGEOP(int8_t, uint8_t, AtomicExchange8SeqCst) +JIT_EXCHANGEOP(uint8_t, uint8_t, AtomicExchange8SeqCst) +JIT_EXCHANGEOP(int16_t, uint16_t, AtomicExchange16SeqCst) +JIT_EXCHANGEOP(uint16_t, uint16_t, AtomicExchange16SeqCst) +JIT_EXCHANGEOP(int32_t, uint32_t, AtomicExchange32SeqCst) +JIT_EXCHANGEOP(uint32_t, uint32_t, AtomicExchange32SeqCst) + +#ifdef JIT_EXCHANGEOP_CAS +JIT_EXCHANGEOP_CAS(int64_t) +JIT_EXCHANGEOP_CAS(uint64_t) +#else +JIT_EXCHANGEOP(int64_t, uint64_t, AtomicExchange64SeqCst) +JIT_EXCHANGEOP(uint64_t, uint64_t, AtomicExchange64SeqCst) +#endif + +} // namespace jit +} // namespace js + +#undef JIT_EXCHANGEOP +#undef JIT_EXCHANGEOP_CAS + +#define JIT_CAS(T, U, cmpxchg) \ + template <> \ + inline T AtomicOperations::compareExchangeSeqCst(T* addr, T oldval, \ + T newval) { \ + return (T)cmpxchg((U*)addr, (U)oldval, (U)newval); \ + } + +namespace js { +namespace jit { + +JIT_CAS(int8_t, uint8_t, AtomicCmpXchg8SeqCst) +JIT_CAS(uint8_t, uint8_t, AtomicCmpXchg8SeqCst) +JIT_CAS(int16_t, uint16_t, AtomicCmpXchg16SeqCst) +JIT_CAS(uint16_t, uint16_t, AtomicCmpXchg16SeqCst) +JIT_CAS(int32_t, uint32_t, AtomicCmpXchg32SeqCst) +JIT_CAS(uint32_t, uint32_t, AtomicCmpXchg32SeqCst) +JIT_CAS(int64_t, uint64_t, AtomicCmpXchg64SeqCst) +JIT_CAS(uint64_t, uint64_t, AtomicCmpXchg64SeqCst) + +} // namespace jit +} // namespace js + +#undef JIT_CAS + +#define JIT_FETCHADDOP(T, U, xadd) \ + template <> \ + inline T AtomicOperations::fetchAddSeqCst(T* addr, T val) { \ + return (T)xadd((U*)addr, (U)val); \ + } + +#define JIT_FETCHSUBOP(T) \ + template <> \ + inline T AtomicOperations::fetchSubSeqCst(T* addr, T val) { \ + return fetchAddSeqCst(addr, (T)(0 - val)); \ + } + +#ifndef JS_64BIT +# define JIT_FETCHADDOP_CAS(T) \ + template <> \ + inline T AtomicOperations::fetchAddSeqCst(T* addr, T val) { \ + AtomicCompilerFence(); \ + T oldval = *addr; /* Good initial approximation */ \ + for (;;) { \ + T nextval = (T)AtomicCmpXchg64SeqCst( \ + (uint64_t*)addr, (uint64_t)oldval, (uint64_t)(oldval + val)); \ + if (nextval == oldval) { \ + break; \ + } \ + oldval = nextval; \ + } \ + AtomicCompilerFence(); \ + return oldval; \ + } +#endif // !JS_64BIT + +namespace js { +namespace jit { + +JIT_FETCHADDOP(int8_t, uint8_t, AtomicAdd8SeqCst) +JIT_FETCHADDOP(uint8_t, uint8_t, AtomicAdd8SeqCst) +JIT_FETCHADDOP(int16_t, uint16_t, AtomicAdd16SeqCst) +JIT_FETCHADDOP(uint16_t, uint16_t, AtomicAdd16SeqCst) +JIT_FETCHADDOP(int32_t, uint32_t, AtomicAdd32SeqCst) +JIT_FETCHADDOP(uint32_t, uint32_t, AtomicAdd32SeqCst) + +#ifdef JIT_FETCHADDOP_CAS +JIT_FETCHADDOP_CAS(int64_t) +JIT_FETCHADDOP_CAS(uint64_t) +#else +JIT_FETCHADDOP(int64_t, uint64_t, AtomicAdd64SeqCst) +JIT_FETCHADDOP(uint64_t, uint64_t, AtomicAdd64SeqCst) +#endif + +JIT_FETCHSUBOP(int8_t) +JIT_FETCHSUBOP(uint8_t) +JIT_FETCHSUBOP(int16_t) +JIT_FETCHSUBOP(uint16_t) +JIT_FETCHSUBOP(int32_t) +JIT_FETCHSUBOP(uint32_t) +JIT_FETCHSUBOP(int64_t) +JIT_FETCHSUBOP(uint64_t) + +} // namespace jit +} // namespace js + +#undef JIT_FETCHADDOP +#undef JIT_FETCHADDOP_CAS +#undef JIT_FETCHSUBOP + +#define JIT_FETCHBITOPX(T, U, name, op) \ + template <> \ + inline T AtomicOperations::name(T* addr, T val) { \ + return (T)op((U*)addr, (U)val); \ + } + +#define JIT_FETCHBITOP(T, U, andop, orop, xorop) \ + JIT_FETCHBITOPX(T, U, fetchAndSeqCst, andop) \ + JIT_FETCHBITOPX(T, U, fetchOrSeqCst, orop) \ + JIT_FETCHBITOPX(T, U, fetchXorSeqCst, xorop) + +#ifndef JS_64BIT + +# define AND_OP & +# define OR_OP | +# define XOR_OP ^ + +# define JIT_FETCHBITOPX_CAS(T, name, OP) \ + template <> \ + inline T AtomicOperations::name(T* addr, T val) { \ + AtomicCompilerFence(); \ + T oldval = *addr; \ + for (;;) { \ + T nextval = (T)AtomicCmpXchg64SeqCst( \ + (uint64_t*)addr, (uint64_t)oldval, (uint64_t)(oldval OP val)); \ + if (nextval == oldval) { \ + break; \ + } \ + oldval = nextval; \ + } \ + AtomicCompilerFence(); \ + return oldval; \ + } + +# define JIT_FETCHBITOP_CAS(T) \ + JIT_FETCHBITOPX_CAS(T, fetchAndSeqCst, AND_OP) \ + JIT_FETCHBITOPX_CAS(T, fetchOrSeqCst, OR_OP) \ + JIT_FETCHBITOPX_CAS(T, fetchXorSeqCst, XOR_OP) + +#endif // !JS_64BIT + +namespace js { +namespace jit { + +JIT_FETCHBITOP(int8_t, uint8_t, AtomicAnd8SeqCst, AtomicOr8SeqCst, + AtomicXor8SeqCst) +JIT_FETCHBITOP(uint8_t, uint8_t, AtomicAnd8SeqCst, AtomicOr8SeqCst, + AtomicXor8SeqCst) +JIT_FETCHBITOP(int16_t, uint16_t, AtomicAnd16SeqCst, AtomicOr16SeqCst, + AtomicXor16SeqCst) +JIT_FETCHBITOP(uint16_t, uint16_t, AtomicAnd16SeqCst, AtomicOr16SeqCst, + AtomicXor16SeqCst) +JIT_FETCHBITOP(int32_t, uint32_t, AtomicAnd32SeqCst, AtomicOr32SeqCst, + AtomicXor32SeqCst) +JIT_FETCHBITOP(uint32_t, uint32_t, AtomicAnd32SeqCst, AtomicOr32SeqCst, + AtomicXor32SeqCst) + +#ifdef JIT_FETCHBITOP_CAS +JIT_FETCHBITOP_CAS(int64_t) +JIT_FETCHBITOP_CAS(uint64_t) +#else +JIT_FETCHBITOP(int64_t, uint64_t, AtomicAnd64SeqCst, AtomicOr64SeqCst, + AtomicXor64SeqCst) +JIT_FETCHBITOP(uint64_t, uint64_t, AtomicAnd64SeqCst, AtomicOr64SeqCst, + AtomicXor64SeqCst) +#endif + +} // namespace jit +} // namespace js + +#undef JIT_FETCHBITOPX_CAS +#undef JIT_FETCHBITOPX +#undef JIT_FETCHBITOP_CAS +#undef JIT_FETCHBITOP + +#define JIT_LOADSAFE(T, U, loadop) \ + template <> \ + inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) { \ + union { \ + U u; \ + T t; \ + }; \ + u = loadop((U*)addr); \ + return t; \ + } + +#ifndef JS_64BIT +# define JIT_LOADSAFE_TEARING(T) \ + template <> \ + inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) { \ + MOZ_ASSERT(sizeof(T) == 8); \ + union { \ + uint32_t u[2]; \ + T t; \ + }; \ + uint32_t* ptr = (uint32_t*)addr; \ + u[0] = AtomicLoad32Unsynchronized(ptr); \ + u[1] = AtomicLoad32Unsynchronized(ptr + 1); \ + return t; \ + } +#endif // !JS_64BIT + +namespace js { +namespace jit { + +JIT_LOADSAFE(int8_t, uint8_t, AtomicLoad8Unsynchronized) +JIT_LOADSAFE(uint8_t, uint8_t, AtomicLoad8Unsynchronized) +JIT_LOADSAFE(int16_t, uint16_t, AtomicLoad16Unsynchronized) +JIT_LOADSAFE(uint16_t, uint16_t, AtomicLoad16Unsynchronized) +JIT_LOADSAFE(int32_t, uint32_t, AtomicLoad32Unsynchronized) +JIT_LOADSAFE(uint32_t, uint32_t, AtomicLoad32Unsynchronized) +#ifdef JIT_LOADSAFE_TEARING +JIT_LOADSAFE_TEARING(int64_t) +JIT_LOADSAFE_TEARING(uint64_t) +JIT_LOADSAFE_TEARING(double) +#else +JIT_LOADSAFE(int64_t, uint64_t, AtomicLoad64Unsynchronized) +JIT_LOADSAFE(uint64_t, uint64_t, AtomicLoad64Unsynchronized) +JIT_LOADSAFE(double, uint64_t, AtomicLoad64Unsynchronized) +#endif +JIT_LOADSAFE(float, uint32_t, AtomicLoad32Unsynchronized) + +// Clang requires a specialization for uint8_clamped. +template <> +inline uint8_clamped js::jit::AtomicOperations::loadSafeWhenRacy( + uint8_clamped* addr) { + return uint8_clamped(loadSafeWhenRacy((uint8_t*)addr)); +} + +} // namespace jit +} // namespace js + +#undef JIT_LOADSAFE +#undef JIT_LOADSAFE_TEARING + +#define JIT_STORESAFE(T, U, storeop) \ + template <> \ + inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) { \ + union { \ + U u; \ + T t; \ + }; \ + t = val; \ + storeop((U*)addr, u); \ + } + +#ifndef JS_64BIT +# define JIT_STORESAFE_TEARING(T) \ + template <> \ + inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) { \ + union { \ + uint32_t u[2]; \ + T t; \ + }; \ + t = val; \ + uint32_t* ptr = (uint32_t*)addr; \ + AtomicStore32Unsynchronized(ptr, u[0]); \ + AtomicStore32Unsynchronized(ptr + 1, u[1]); \ + } +#endif // !JS_64BIT + +namespace js { +namespace jit { + +JIT_STORESAFE(int8_t, uint8_t, AtomicStore8Unsynchronized) +JIT_STORESAFE(uint8_t, uint8_t, AtomicStore8Unsynchronized) +JIT_STORESAFE(int16_t, uint16_t, AtomicStore16Unsynchronized) +JIT_STORESAFE(uint16_t, uint16_t, AtomicStore16Unsynchronized) +JIT_STORESAFE(int32_t, uint32_t, AtomicStore32Unsynchronized) +JIT_STORESAFE(uint32_t, uint32_t, AtomicStore32Unsynchronized) +#ifdef JIT_STORESAFE_TEARING +JIT_STORESAFE_TEARING(int64_t) +JIT_STORESAFE_TEARING(uint64_t) +JIT_STORESAFE_TEARING(double) +#else +JIT_STORESAFE(int64_t, uint64_t, AtomicStore64Unsynchronized) +JIT_STORESAFE(uint64_t, uint64_t, AtomicStore64Unsynchronized) +JIT_STORESAFE(double, uint64_t, AtomicStore64Unsynchronized) +#endif +JIT_STORESAFE(float, uint32_t, AtomicStore32Unsynchronized) + +// Clang requires a specialization for uint8_clamped. +template <> +inline void js::jit::AtomicOperations::storeSafeWhenRacy(uint8_clamped* addr, + uint8_clamped val) { + storeSafeWhenRacy((uint8_t*)addr, (uint8_t)val); +} + +} // namespace jit +} // namespace js + +#undef JIT_STORESAFE +#undef JIT_STORESAFE_TEARING + +void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest, const void* src, + size_t nbytes) { + MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes)); + MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes)); + AtomicMemcpyDownUnsynchronized((uint8_t*)dest, (const uint8_t*)src, nbytes); +} + +inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest, + const void* src, + size_t nbytes) { + if ((char*)dest <= (char*)src) { + AtomicMemcpyDownUnsynchronized((uint8_t*)dest, (const uint8_t*)src, nbytes); + } else { + AtomicMemcpyUpUnsynchronized((uint8_t*)dest, (const uint8_t*)src, nbytes); + } +} + +#endif // jit_shared_AtomicOperations_shared_jit_h diff --git a/js/src/jit/shared/CodeGenerator-shared-inl.h b/js/src/jit/shared/CodeGenerator-shared-inl.h new file mode 100644 index 0000000000..1c9880f59f --- /dev/null +++ b/js/src/jit/shared/CodeGenerator-shared-inl.h @@ -0,0 +1,342 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_CodeGenerator_shared_inl_h +#define jit_shared_CodeGenerator_shared_inl_h + +#include "jit/shared/CodeGenerator-shared.h" + +#include "jit/JitFrames.h" +#include "jit/ScalarTypeUtils.h" + +#include "jit/MacroAssembler-inl.h" + +namespace js { +namespace jit { + +static inline bool IsConstant(const LInt64Allocation& a) { +#if JS_BITS_PER_WORD == 32 + if (a.high().isConstantValue()) { + return true; + } + if (a.high().isConstantIndex()) { + return true; + } +#else + if (a.value().isConstantValue()) { + return true; + } + if (a.value().isConstantIndex()) { + return true; + } +#endif + return false; +} + +static inline int32_t ToInt32(const LAllocation* a) { + if (a->isConstantValue()) { + const MConstant* cst = a->toConstant(); + if (cst->type() == MIRType::Int32) { + return cst->toInt32(); + } + intptr_t val = cst->toIntPtr(); + MOZ_ASSERT(INT32_MIN <= val && val <= INT32_MAX); + return int32_t(val); + } + if (a->isConstantIndex()) { + return a->toConstantIndex()->index(); + } + MOZ_CRASH("this is not a constant!"); +} + +static inline int64_t ToInt64(const LAllocation* a) { + if (a->isConstantValue()) { + return a->toConstant()->toInt64(); + } + if (a->isConstantIndex()) { + return a->toConstantIndex()->index(); + } + MOZ_CRASH("this is not a constant!"); +} + +static inline int64_t ToInt64(const LInt64Allocation& a) { +#if JS_BITS_PER_WORD == 32 + if (a.high().isConstantValue()) { + return a.high().toConstant()->toInt64(); + } + if (a.high().isConstantIndex()) { + return a.high().toConstantIndex()->index(); + } +#else + if (a.value().isConstantValue()) { + return a.value().toConstant()->toInt64(); + } + if (a.value().isConstantIndex()) { + return a.value().toConstantIndex()->index(); + } +#endif + MOZ_CRASH("this is not a constant!"); +} + +static inline double ToDouble(const LAllocation* a) { + return a->toConstant()->numberToDouble(); +} + +static inline bool ToBoolean(const LAllocation* a) { + return a->toConstant()->toBoolean(); +} + +static inline Register ToRegister(const LAllocation& a) { + MOZ_ASSERT(a.isGeneralReg()); + return a.toGeneralReg()->reg(); +} + +static inline Register ToRegister(const LAllocation* a) { + return ToRegister(*a); +} + +static inline Register ToRegister(const LDefinition* def) { + return ToRegister(*def->output()); +} + +static inline Register64 ToOutRegister64(LInstruction* ins) { +#if JS_BITS_PER_WORD == 32 + Register loReg = ToRegister(ins->getDef(INT64LOW_INDEX)); + Register hiReg = ToRegister(ins->getDef(INT64HIGH_INDEX)); + return Register64(hiReg, loReg); +#else + return Register64(ToRegister(ins->getDef(0))); +#endif +} + +static inline Register64 ToRegister64(const LInt64Allocation& a) { +#if JS_BITS_PER_WORD == 32 + return Register64(ToRegister(a.high()), ToRegister(a.low())); +#else + return Register64(ToRegister(a.value())); +#endif +} + +static inline Register64 ToRegister64(const LInt64Definition& a) { +#if JS_BITS_PER_WORD == 32 + return Register64(ToRegister(a.pointerHigh()), ToRegister(a.pointerLow())); +#else + return Register64(ToRegister(a.pointer())); +#endif +} + +static inline Register ToTempRegisterOrInvalid(const LDefinition* def) { + if (def->isBogusTemp()) { + return InvalidReg; + } + return ToRegister(def); +} + +static inline Register64 ToTempRegister64OrInvalid( + const LInt64Definition& def) { + if (def.isBogusTemp()) { + return Register64::Invalid(); + } + return ToRegister64(def); +} + +static inline Register ToTempUnboxRegister(const LDefinition* def) { + return ToTempRegisterOrInvalid(def); +} + +static inline Register ToRegisterOrInvalid(const LDefinition* a) { + return a ? ToRegister(a) : InvalidReg; +} + +static inline FloatRegister ToFloatRegister(const LAllocation& a) { + MOZ_ASSERT(a.isFloatReg()); + return a.toFloatReg()->reg(); +} + +static inline FloatRegister ToFloatRegister(const LAllocation* a) { + return ToFloatRegister(*a); +} + +static inline FloatRegister ToFloatRegister(const LDefinition* def) { + return ToFloatRegister(*def->output()); +} + +static inline FloatRegister ToTempFloatRegisterOrInvalid( + const LDefinition* def) { + if (def->isBogusTemp()) { + return InvalidFloatReg; + } + return ToFloatRegister(def); +} + +static inline AnyRegister ToAnyRegister(const LAllocation& a) { + MOZ_ASSERT(a.isGeneralReg() || a.isFloatReg()); + if (a.isGeneralReg()) { + return AnyRegister(ToRegister(a)); + } + return AnyRegister(ToFloatRegister(a)); +} + +static inline AnyRegister ToAnyRegister(const LAllocation* a) { + return ToAnyRegister(*a); +} + +static inline AnyRegister ToAnyRegister(const LDefinition* def) { + return ToAnyRegister(def->output()); +} + +static inline ValueOperand ToOutValue(LInstruction* ins) { +#if defined(JS_NUNBOX32) + return ValueOperand(ToRegister(ins->getDef(TYPE_INDEX)), + ToRegister(ins->getDef(PAYLOAD_INDEX))); +#elif defined(JS_PUNBOX64) + return ValueOperand(ToRegister(ins->getDef(0))); +#else +# error "Unknown" +#endif +} + +static inline ValueOperand GetTempValue(Register type, Register payload) { +#if defined(JS_NUNBOX32) + return ValueOperand(type, payload); +#elif defined(JS_PUNBOX64) + (void)type; + return ValueOperand(payload); +#else +# error "Unknown" +#endif +} + +// For argument construction for calls. Argslots are Value-sized. +Address CodeGeneratorShared::AddressOfPassedArg(uint32_t slot) const { + MOZ_ASSERT(masm.framePushed() == frameSize()); + + MOZ_ASSERT(slot > 0); + MOZ_ASSERT(slot <= graph.argumentSlotCount()); + + uint32_t offsetFromBase = offsetOfPassedArgSlots_ + slot * sizeof(Value); + MOZ_ASSERT(offsetFromBase <= frameSize()); + + // Space for passed arguments is reserved below a function's local stack + // storage. Note that passedArgSlotsOffset_ is aligned to at least + // sizeof(Value) to ensure proper alignment. + MOZ_ASSERT((offsetFromBase % sizeof(Value)) == 0); + + if (JitOptions.baseRegForLocals == BaseRegForAddress::SP) { + return Address(masm.getStackPointer(), frameSize() - offsetFromBase); + } + MOZ_ASSERT(JitOptions.baseRegForLocals == BaseRegForAddress::FP); + return Address(FramePointer, -int32_t(offsetFromBase)); +} + +uint32_t CodeGeneratorShared::UnusedStackBytesForCall( + uint32_t numArgSlots) const { + MOZ_ASSERT(masm.framePushed() == frameSize()); + MOZ_ASSERT(numArgSlots <= graph.argumentSlotCount()); + uint32_t unusedArgSlots = graph.argumentSlotCount() - numArgSlots; + return unusedArgSlots * sizeof(Value); +} + +template <BaseRegForAddress Base> +Address CodeGeneratorShared::ToAddress(const LAllocation& a) const { + MOZ_ASSERT(a.isMemory() || a.isStackArea()); + MOZ_ASSERT(masm.framePushed() == frameSize()); + + if (a.isArgument()) { + // Use the frame pointer, unless the caller explicitly requested a + // stack-pointer-relative address. + uint32_t offsetFromFP = offsetOfArgsFromFP_ + a.toArgument()->index(); + if constexpr (Base == BaseRegForAddress::SP) { + return Address(masm.getStackPointer(), frameSize() + offsetFromFP); + } else { + static_assert(Base == BaseRegForAddress::Default || + Base == BaseRegForAddress::FP); + return Address(FramePointer, offsetFromFP); + } + } + + uint32_t slot = + a.isStackSlot() ? a.toStackSlot()->slot() : a.toStackArea()->base(); + MOZ_ASSERT(slot > 0 && slot <= graph.localSlotsSize()); + MOZ_ASSERT(slot <= frameSize()); + + BaseRegForAddress base = Base; + if constexpr (Base == BaseRegForAddress::Default) { + base = JitOptions.baseRegForLocals; + } + + if (base == BaseRegForAddress::FP) { + return Address(FramePointer, -int32_t(slot)); + } + MOZ_ASSERT(base == BaseRegForAddress::SP); + return Address(masm.getStackPointer(), frameSize() - slot); +} + +template <BaseRegForAddress Base> +Address CodeGeneratorShared::ToAddress(const LAllocation* a) const { + return ToAddress<Base>(*a); +} + +// static +Address CodeGeneratorShared::ToAddress(Register elements, + const LAllocation* index, + Scalar::Type type, + int32_t offsetAdjustment) { + int32_t idx = ToInt32(index); + int32_t offset; + MOZ_ALWAYS_TRUE(ArrayOffsetFitsInInt32(idx, type, offsetAdjustment, &offset)); + return Address(elements, offset); +} + +void CodeGeneratorShared::saveLive(LInstruction* ins) { + MOZ_ASSERT(!ins->isCall()); + LSafepoint* safepoint = ins->safepoint(); + masm.PushRegsInMask(safepoint->liveRegs()); +} + +void CodeGeneratorShared::restoreLive(LInstruction* ins) { + MOZ_ASSERT(!ins->isCall()); + LSafepoint* safepoint = ins->safepoint(); + masm.PopRegsInMask(safepoint->liveRegs()); +} + +void CodeGeneratorShared::restoreLiveIgnore(LInstruction* ins, + LiveRegisterSet ignore) { + MOZ_ASSERT(!ins->isCall()); + LSafepoint* safepoint = ins->safepoint(); + masm.PopRegsInMaskIgnore(safepoint->liveRegs(), ignore); +} + +LiveRegisterSet CodeGeneratorShared::liveVolatileRegs(LInstruction* ins) { + MOZ_ASSERT(!ins->isCall()); + LSafepoint* safepoint = ins->safepoint(); + LiveRegisterSet regs; + regs.set() = RegisterSet::Intersect(safepoint->liveRegs().set(), + RegisterSet::Volatile()); + return regs; +} + +void CodeGeneratorShared::saveLiveVolatile(LInstruction* ins) { + LiveRegisterSet regs = liveVolatileRegs(ins); + masm.PushRegsInMask(regs); +} + +void CodeGeneratorShared::restoreLiveVolatile(LInstruction* ins) { + LiveRegisterSet regs = liveVolatileRegs(ins); + masm.PopRegsInMask(regs); +} + +inline bool CodeGeneratorShared::isGlobalObject(JSObject* object) { + // Calling object->is<GlobalObject>() is racy because this relies on + // checking the group and this can be changed while we are compiling off the + // main thread. Note that we only check for the script realm's global here. + return object == gen->realm->maybeGlobal(); +} + +} // namespace jit +} // namespace js + +#endif /* jit_shared_CodeGenerator_shared_inl_h */ diff --git a/js/src/jit/shared/CodeGenerator-shared.cpp b/js/src/jit/shared/CodeGenerator-shared.cpp new file mode 100644 index 0000000000..04a8baa752 --- /dev/null +++ b/js/src/jit/shared/CodeGenerator-shared.cpp @@ -0,0 +1,983 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/shared/CodeGenerator-shared-inl.h" + +#include "mozilla/DebugOnly.h" + +#include <utility> + +#include "jit/CodeGenerator.h" +#include "jit/CompactBuffer.h" +#include "jit/CompileInfo.h" +#include "jit/InlineScriptTree.h" +#include "jit/JitcodeMap.h" +#include "jit/JitFrames.h" +#include "jit/JitSpewer.h" +#include "jit/MacroAssembler.h" +#include "jit/MIR.h" +#include "jit/MIRGenerator.h" +#include "jit/SafepointIndex.h" +#include "js/Conversions.h" +#include "util/Memory.h" + +#include "jit/MacroAssembler-inl.h" +#include "vm/JSScript-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::BitwiseCast; +using mozilla::DebugOnly; + +namespace js { +namespace jit { + +MacroAssembler& CodeGeneratorShared::ensureMasm(MacroAssembler* masmArg, + TempAllocator& alloc, + CompileRealm* realm) { + if (masmArg) { + return *masmArg; + } + maybeMasm_.emplace(alloc, realm); + return *maybeMasm_; +} + +CodeGeneratorShared::CodeGeneratorShared(MIRGenerator* gen, LIRGraph* graph, + MacroAssembler* masmArg) + : maybeMasm_(), + masm(ensureMasm(masmArg, gen->alloc(), gen->realm)), + gen(gen), + graph(*graph), + current(nullptr), + snapshots_(), + recovers_(), +#ifdef DEBUG + pushedArgs_(0), +#endif + lastOsiPointOffset_(0), + safepoints_(graph->localSlotsSize(), + (gen->outerInfo().nargs() + 1) * sizeof(Value)), + returnLabel_(), + nativeToBytecodeMap_(nullptr), + nativeToBytecodeMapSize_(0), + nativeToBytecodeTableOffset_(0), +#ifdef CHECK_OSIPOINT_REGISTERS + checkOsiPointRegisters(JitOptions.checkOsiPointRegisters), +#endif + frameDepth_(0) { + if (gen->isProfilerInstrumentationEnabled()) { + masm.enableProfilingInstrumentation(); + } + + if (gen->compilingWasm()) { + offsetOfArgsFromFP_ = sizeof(wasm::Frame); + +#ifdef JS_CODEGEN_ARM64 + // Ensure SP is aligned to 16 bytes. + frameDepth_ = AlignBytes(graph->localSlotsSize(), WasmStackAlignment); +#else + frameDepth_ = AlignBytes(graph->localSlotsSize(), sizeof(uintptr_t)); +#endif + +#ifdef ENABLE_WASM_SIMD +# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \ + defined(JS_CODEGEN_ARM64) + // On X64/x86 and ARM64, we don't need alignment for Wasm SIMD at this time. +# else +# error \ + "we may need padding so that local slots are SIMD-aligned and the stack must be kept SIMD-aligned too." +# endif +#endif + + if (gen->needsStaticStackAlignment()) { + // Since wasm uses the system ABI which does not necessarily use a + // regular array where all slots are sizeof(Value), it maintains the max + // argument stack depth separately. + MOZ_ASSERT(graph->argumentSlotCount() == 0); + frameDepth_ += gen->wasmMaxStackArgBytes(); + + // An MWasmCall does not align the stack pointer at calls sites but + // instead relies on the a priori stack adjustment. This must be the + // last adjustment of frameDepth_. + frameDepth_ += ComputeByteAlignment(sizeof(wasm::Frame) + frameDepth_, + WasmStackAlignment); + } + +#ifdef JS_CODEGEN_ARM64 + MOZ_ASSERT((frameDepth_ % WasmStackAlignment) == 0, + "Trap exit stub needs 16-byte aligned stack pointer"); +#endif + } else { + offsetOfArgsFromFP_ = sizeof(JitFrameLayout); + + // Allocate space for local slots (register allocator spills). Round to + // JitStackAlignment, and implicitly to sizeof(Value) as JitStackAlignment + // is a multiple of sizeof(Value). This was originally implemented for + // SIMD.js, but now lets us use faster ABI calls via setupAlignedABICall. + frameDepth_ = AlignBytes(graph->localSlotsSize(), JitStackAlignment); + + // Allocate space for argument Values passed to callee functions. + offsetOfPassedArgSlots_ = frameDepth_; + MOZ_ASSERT((offsetOfPassedArgSlots_ % sizeof(JS::Value)) == 0); + frameDepth_ += graph->argumentSlotCount() * sizeof(JS::Value); + + MOZ_ASSERT((frameDepth_ % JitStackAlignment) == 0); + } +} + +bool CodeGeneratorShared::generatePrologue() { + MOZ_ASSERT(masm.framePushed() == 0); + MOZ_ASSERT(!gen->compilingWasm()); + +#ifdef JS_USE_LINK_REGISTER + masm.pushReturnAddress(); +#endif + + // Frame prologue. + masm.push(FramePointer); + masm.moveStackPtrTo(FramePointer); + + // Ensure that the Ion frame is properly aligned. + masm.assertStackAlignment(JitStackAlignment, 0); + + // If profiling, save the current frame pointer to a per-thread global field. + if (isProfilerInstrumentationEnabled()) { + masm.profilerEnterFrame(FramePointer, CallTempReg0); + } + + // Note that this automatically sets MacroAssembler::framePushed(). + masm.reserveStack(frameSize()); + MOZ_ASSERT(masm.framePushed() == frameSize()); + masm.checkStackAlignment(); + + return true; +} + +bool CodeGeneratorShared::generateEpilogue() { + MOZ_ASSERT(!gen->compilingWasm()); + masm.bind(&returnLabel_); + + // If profiling, jump to a trampoline to reset the JitActivation's + // lastProfilingFrame to point to the previous frame and return to the caller. + if (isProfilerInstrumentationEnabled()) { + masm.profilerExitFrame(); + } + + MOZ_ASSERT(masm.framePushed() == frameSize()); + masm.moveToStackPtr(FramePointer); + masm.pop(FramePointer); + masm.setFramePushed(0); + + masm.ret(); + + // On systems that use a constant pool, this is a good time to emit. + masm.flushBuffer(); + return true; +} + +bool CodeGeneratorShared::generateOutOfLineCode() { + AutoCreatedBy acb(masm, "CodeGeneratorShared::generateOutOfLineCode"); + + // OOL paths should not attempt to use |current| as it's the last block + // instead of the block corresponding to the OOL path. + current = nullptr; + + for (size_t i = 0; i < outOfLineCode_.length(); i++) { + // Add native => bytecode mapping entries for OOL sites. + // Not enabled on wasm yet since it doesn't contain bytecode mappings. + if (!gen->compilingWasm()) { + if (!addNativeToBytecodeEntry(outOfLineCode_[i]->bytecodeSite())) { + return false; + } + } + + if (!gen->alloc().ensureBallast()) { + return false; + } + + JitSpew(JitSpew_Codegen, "# Emitting out of line code"); + + masm.setFramePushed(outOfLineCode_[i]->framePushed()); + outOfLineCode_[i]->bind(&masm); + + outOfLineCode_[i]->generate(this); + } + + return !masm.oom(); +} + +void CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code, + const MInstruction* mir) { + MOZ_ASSERT(mir); + addOutOfLineCode(code, mir->trackedSite()); +} + +void CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code, + const BytecodeSite* site) { + MOZ_ASSERT_IF(!gen->compilingWasm(), site->script()->containsPC(site->pc())); + code->setFramePushed(masm.framePushed()); + code->setBytecodeSite(site); + masm.propagateOOM(outOfLineCode_.append(code)); +} + +bool CodeGeneratorShared::addNativeToBytecodeEntry(const BytecodeSite* site) { + MOZ_ASSERT(site); + MOZ_ASSERT(site->tree()); + MOZ_ASSERT(site->pc()); + + // Skip the table entirely if profiling is not enabled. + if (!isProfilerInstrumentationEnabled()) { + return true; + } + + // Fails early if the last added instruction caused the macro assembler to + // run out of memory as continuity assumption below do not hold. + if (masm.oom()) { + return false; + } + + InlineScriptTree* tree = site->tree(); + jsbytecode* pc = site->pc(); + uint32_t nativeOffset = masm.currentOffset(); + + MOZ_ASSERT_IF(nativeToBytecodeList_.empty(), nativeOffset == 0); + + if (!nativeToBytecodeList_.empty()) { + size_t lastIdx = nativeToBytecodeList_.length() - 1; + NativeToBytecode& lastEntry = nativeToBytecodeList_[lastIdx]; + + MOZ_ASSERT(nativeOffset >= lastEntry.nativeOffset.offset()); + + // If the new entry is for the same inlineScriptTree and same + // bytecodeOffset, but the nativeOffset has changed, do nothing. + // The same site just generated some more code. + if (lastEntry.tree == tree && lastEntry.pc == pc) { + JitSpew(JitSpew_Profiling, " => In-place update [%zu-%" PRIu32 "]", + lastEntry.nativeOffset.offset(), nativeOffset); + return true; + } + + // If the new entry is for the same native offset, then update the + // previous entry with the new bytecode site, since the previous + // bytecode site did not generate any native code. + if (lastEntry.nativeOffset.offset() == nativeOffset) { + lastEntry.tree = tree; + lastEntry.pc = pc; + JitSpew(JitSpew_Profiling, " => Overwriting zero-length native region."); + + // This overwrite might have made the entry merge-able with a + // previous one. If so, merge it. + if (lastIdx > 0) { + NativeToBytecode& nextToLastEntry = nativeToBytecodeList_[lastIdx - 1]; + if (nextToLastEntry.tree == lastEntry.tree && + nextToLastEntry.pc == lastEntry.pc) { + JitSpew(JitSpew_Profiling, " => Merging with previous region"); + nativeToBytecodeList_.erase(&lastEntry); + } + } + + dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1); + return true; + } + } + + // Otherwise, some native code was generated for the previous bytecode site. + // Add a new entry for code that is about to be generated. + NativeToBytecode entry; + entry.nativeOffset = CodeOffset(nativeOffset); + entry.tree = tree; + entry.pc = pc; + if (!nativeToBytecodeList_.append(entry)) { + return false; + } + + JitSpew(JitSpew_Profiling, " => Push new entry."); + dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1); + return true; +} + +void CodeGeneratorShared::dumpNativeToBytecodeEntries() { +#ifdef JS_JITSPEW + InlineScriptTree* topTree = gen->outerInfo().inlineScriptTree(); + JitSpewStart(JitSpew_Profiling, "Native To Bytecode Entries for %s:%u:%u\n", + topTree->script()->filename(), topTree->script()->lineno(), + topTree->script()->column()); + for (unsigned i = 0; i < nativeToBytecodeList_.length(); i++) { + dumpNativeToBytecodeEntry(i); + } +#endif +} + +void CodeGeneratorShared::dumpNativeToBytecodeEntry(uint32_t idx) { +#ifdef JS_JITSPEW + NativeToBytecode& ref = nativeToBytecodeList_[idx]; + InlineScriptTree* tree = ref.tree; + JSScript* script = tree->script(); + uint32_t nativeOffset = ref.nativeOffset.offset(); + unsigned nativeDelta = 0; + unsigned pcDelta = 0; + if (idx + 1 < nativeToBytecodeList_.length()) { + NativeToBytecode* nextRef = &ref + 1; + nativeDelta = nextRef->nativeOffset.offset() - nativeOffset; + if (nextRef->tree == ref.tree) { + pcDelta = nextRef->pc - ref.pc; + } + } + JitSpewStart( + JitSpew_Profiling, " %08zx [+%-6u] => %-6ld [%-4u] {%-10s} (%s:%u:%u", + ref.nativeOffset.offset(), nativeDelta, (long)(ref.pc - script->code()), + pcDelta, CodeName(JSOp(*ref.pc)), script->filename(), script->lineno(), + script->column()); + + for (tree = tree->caller(); tree; tree = tree->caller()) { + JitSpewCont(JitSpew_Profiling, " <= %s:%u:%u", tree->script()->filename(), + tree->script()->lineno(), tree->script()->column()); + } + JitSpewCont(JitSpew_Profiling, ")"); + JitSpewFin(JitSpew_Profiling); +#endif +} + +// see OffsetOfFrameSlot +static inline int32_t ToStackIndex(LAllocation* a) { + if (a->isStackSlot()) { + MOZ_ASSERT(a->toStackSlot()->slot() >= 1); + return a->toStackSlot()->slot(); + } + return -int32_t(sizeof(JitFrameLayout) + a->toArgument()->index()); +} + +void CodeGeneratorShared::encodeAllocation(LSnapshot* snapshot, + MDefinition* mir, + uint32_t* allocIndex) { + if (mir->isBox()) { + mir = mir->toBox()->getOperand(0); + } + + MIRType type = mir->isRecoveredOnBailout() ? MIRType::None + : mir->isUnused() ? MIRType::MagicOptimizedOut + : mir->type(); + + RValueAllocation alloc; + + switch (type) { + case MIRType::None: { + MOZ_ASSERT(mir->isRecoveredOnBailout()); + uint32_t index = 0; + LRecoverInfo* recoverInfo = snapshot->recoverInfo(); + MNode** it = recoverInfo->begin(); + MNode** end = recoverInfo->end(); + while (it != end && mir != *it) { + ++it; + ++index; + } + + // This MDefinition is recovered, thus it should be listed in the + // LRecoverInfo. + MOZ_ASSERT(it != end && mir == *it); + + // Lambda should have a default value readable for iterating over the + // inner frames. + MConstant* functionOperand = nullptr; + if (mir->isLambda()) { + functionOperand = mir->toLambda()->functionOperand(); + } else if (mir->isFunctionWithProto()) { + functionOperand = mir->toFunctionWithProto()->functionOperand(); + } + if (functionOperand) { + uint32_t cstIndex; + masm.propagateOOM( + graph.addConstantToPool(functionOperand->toJSValue(), &cstIndex)); + alloc = RValueAllocation::RecoverInstruction(index, cstIndex); + break; + } + + alloc = RValueAllocation::RecoverInstruction(index); + break; + } + case MIRType::Undefined: + alloc = RValueAllocation::Undefined(); + break; + case MIRType::Null: + alloc = RValueAllocation::Null(); + break; + case MIRType::Int32: + case MIRType::String: + case MIRType::Symbol: + case MIRType::BigInt: + case MIRType::Object: + case MIRType::Shape: + case MIRType::Boolean: + case MIRType::Double: { + LAllocation* payload = snapshot->payloadOfSlot(*allocIndex); + if (payload->isConstant()) { + MConstant* constant = mir->toConstant(); + uint32_t index; + masm.propagateOOM( + graph.addConstantToPool(constant->toJSValue(), &index)); + alloc = RValueAllocation::ConstantPool(index); + break; + } + + JSValueType valueType = ValueTypeFromMIRType(type); + + MOZ_DIAGNOSTIC_ASSERT(payload->isMemory() || payload->isRegister()); + if (payload->isMemory()) { + alloc = RValueAllocation::Typed(valueType, ToStackIndex(payload)); + } else if (payload->isGeneralReg()) { + alloc = RValueAllocation::Typed(valueType, ToRegister(payload)); + } else if (payload->isFloatReg()) { + alloc = RValueAllocation::Double(ToFloatRegister(payload)); + } else { + MOZ_CRASH("Unexpected payload type."); + } + break; + } + case MIRType::Float32: + case MIRType::Simd128: { + LAllocation* payload = snapshot->payloadOfSlot(*allocIndex); + if (payload->isConstant()) { + MConstant* constant = mir->toConstant(); + uint32_t index; + masm.propagateOOM( + graph.addConstantToPool(constant->toJSValue(), &index)); + alloc = RValueAllocation::ConstantPool(index); + break; + } + + MOZ_ASSERT(payload->isMemory() || payload->isFloatReg()); + if (payload->isFloatReg()) { + alloc = RValueAllocation::AnyFloat(ToFloatRegister(payload)); + } else { + alloc = RValueAllocation::AnyFloat(ToStackIndex(payload)); + } + break; + } + case MIRType::MagicOptimizedOut: + case MIRType::MagicUninitializedLexical: + case MIRType::MagicIsConstructing: { + uint32_t index; + JSWhyMagic why = JS_GENERIC_MAGIC; + switch (type) { + case MIRType::MagicOptimizedOut: + why = JS_OPTIMIZED_OUT; + break; + case MIRType::MagicUninitializedLexical: + why = JS_UNINITIALIZED_LEXICAL; + break; + case MIRType::MagicIsConstructing: + why = JS_IS_CONSTRUCTING; + break; + default: + MOZ_CRASH("Invalid Magic MIRType"); + } + + Value v = MagicValue(why); + masm.propagateOOM(graph.addConstantToPool(v, &index)); + alloc = RValueAllocation::ConstantPool(index); + break; + } + default: { + MOZ_ASSERT(mir->type() == MIRType::Value); + LAllocation* payload = snapshot->payloadOfSlot(*allocIndex); +#ifdef JS_NUNBOX32 + LAllocation* type = snapshot->typeOfSlot(*allocIndex); + if (type->isRegister()) { + if (payload->isRegister()) { + alloc = + RValueAllocation::Untyped(ToRegister(type), ToRegister(payload)); + } else { + alloc = RValueAllocation::Untyped(ToRegister(type), + ToStackIndex(payload)); + } + } else { + if (payload->isRegister()) { + alloc = RValueAllocation::Untyped(ToStackIndex(type), + ToRegister(payload)); + } else { + alloc = RValueAllocation::Untyped(ToStackIndex(type), + ToStackIndex(payload)); + } + } +#elif JS_PUNBOX64 + if (payload->isRegister()) { + alloc = RValueAllocation::Untyped(ToRegister(payload)); + } else { + alloc = RValueAllocation::Untyped(ToStackIndex(payload)); + } +#endif + break; + } + } + MOZ_DIAGNOSTIC_ASSERT(alloc.valid()); + + // This set an extra bit as part of the RValueAllocation, such that we know + // that recover instruction have to be executed without wrapping the + // instruction in a no-op recover instruction. + if (mir->isIncompleteObject()) { + alloc.setNeedSideEffect(); + } + + masm.propagateOOM(snapshots_.add(alloc)); + + *allocIndex += mir->isRecoveredOnBailout() ? 0 : 1; +} + +void CodeGeneratorShared::encode(LRecoverInfo* recover) { + if (recover->recoverOffset() != INVALID_RECOVER_OFFSET) { + return; + } + + uint32_t numInstructions = recover->numInstructions(); + JitSpew(JitSpew_IonSnapshots, + "Encoding LRecoverInfo %p (frameCount %u, instructions %u)", + (void*)recover, recover->mir()->frameCount(), numInstructions); + + RecoverOffset offset = recovers_.startRecover(numInstructions); + + for (MNode* insn : *recover) { + recovers_.writeInstruction(insn); + } + + recovers_.endRecover(); + recover->setRecoverOffset(offset); + masm.propagateOOM(!recovers_.oom()); +} + +void CodeGeneratorShared::encode(LSnapshot* snapshot) { + if (snapshot->snapshotOffset() != INVALID_SNAPSHOT_OFFSET) { + return; + } + + LRecoverInfo* recoverInfo = snapshot->recoverInfo(); + encode(recoverInfo); + + RecoverOffset recoverOffset = recoverInfo->recoverOffset(); + MOZ_ASSERT(recoverOffset != INVALID_RECOVER_OFFSET); + + JitSpew(JitSpew_IonSnapshots, "Encoding LSnapshot %p (LRecover %p)", + (void*)snapshot, (void*)recoverInfo); + + SnapshotOffset offset = + snapshots_.startSnapshot(recoverOffset, snapshot->bailoutKind()); + +#ifdef TRACK_SNAPSHOTS + uint32_t pcOpcode = 0; + uint32_t lirOpcode = 0; + uint32_t lirId = 0; + uint32_t mirOpcode = 0; + uint32_t mirId = 0; + + if (LInstruction* ins = instruction()) { + lirOpcode = uint32_t(ins->op()); + lirId = ins->id(); + if (MDefinition* mir = ins->mirRaw()) { + mirOpcode = uint32_t(mir->op()); + mirId = mir->id(); + if (jsbytecode* pc = mir->trackedSite()->pc()) { + pcOpcode = *pc; + } + } + } + snapshots_.trackSnapshot(pcOpcode, mirOpcode, mirId, lirOpcode, lirId); +#endif + + uint32_t allocIndex = 0; + for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) { + DebugOnly<uint32_t> allocWritten = snapshots_.allocWritten(); + encodeAllocation(snapshot, *it, &allocIndex); + MOZ_ASSERT_IF(!snapshots_.oom(), + allocWritten + 1 == snapshots_.allocWritten()); + } + + MOZ_ASSERT(allocIndex == snapshot->numSlots()); + snapshots_.endSnapshot(); + snapshot->setSnapshotOffset(offset); + masm.propagateOOM(!snapshots_.oom()); +} + +bool CodeGeneratorShared::encodeSafepoints() { + for (CodegenSafepointIndex& index : safepointIndices_) { + LSafepoint* safepoint = index.safepoint(); + + if (!safepoint->encoded()) { + safepoints_.encode(safepoint); + } + } + + return !safepoints_.oom(); +} + +bool CodeGeneratorShared::createNativeToBytecodeScriptList( + JSContext* cx, IonEntry::ScriptList& scripts) { + MOZ_ASSERT(scripts.empty()); + + InlineScriptTree* tree = gen->outerInfo().inlineScriptTree(); + for (;;) { + // Add script from current tree. + bool found = false; + for (uint32_t i = 0; i < scripts.length(); i++) { + if (scripts[i].script == tree->script()) { + found = true; + break; + } + } + if (!found) { + UniqueChars str = + GeckoProfilerRuntime::allocProfileString(cx, tree->script()); + if (!str) { + return false; + } + if (!scripts.emplaceBack(tree->script(), std::move(str))) { + return false; + } + } + + // Process rest of tree + + // If children exist, emit children. + if (tree->hasChildren()) { + tree = tree->firstChild(); + continue; + } + + // Otherwise, find the first tree up the chain (including this one) + // that contains a next sibling. + while (!tree->hasNextCallee() && tree->hasCaller()) { + tree = tree->caller(); + } + + // If we found a sibling, use it. + if (tree->hasNextCallee()) { + tree = tree->nextCallee(); + continue; + } + + // Otherwise, we must have reached the top without finding any siblings. + MOZ_ASSERT(tree->isOutermostCaller()); + break; + } + + return true; +} + +bool CodeGeneratorShared::generateCompactNativeToBytecodeMap( + JSContext* cx, JitCode* code, IonEntry::ScriptList& scripts) { + MOZ_ASSERT(nativeToBytecodeMap_ == nullptr); + MOZ_ASSERT(nativeToBytecodeMapSize_ == 0); + MOZ_ASSERT(nativeToBytecodeTableOffset_ == 0); + + if (!createNativeToBytecodeScriptList(cx, scripts)) { + return false; + } + + CompactBufferWriter writer; + uint32_t tableOffset = 0; + uint32_t numRegions = 0; + + if (!JitcodeIonTable::WriteIonTable( + writer, scripts, &nativeToBytecodeList_[0], + &nativeToBytecodeList_[0] + nativeToBytecodeList_.length(), + &tableOffset, &numRegions)) { + return false; + } + + MOZ_ASSERT(tableOffset > 0); + MOZ_ASSERT(numRegions > 0); + + // Writer is done, copy it to sized buffer. + uint8_t* data = cx->pod_malloc<uint8_t>(writer.length()); + if (!data) { + return false; + } + + memcpy(data, writer.buffer(), writer.length()); + nativeToBytecodeMap_.reset(data); + nativeToBytecodeMapSize_ = writer.length(); + nativeToBytecodeTableOffset_ = tableOffset; + + verifyCompactNativeToBytecodeMap(code, scripts, numRegions); + + JitSpew(JitSpew_Profiling, "Compact Native To Bytecode Map [%p-%p]", data, + data + nativeToBytecodeMapSize_); + + return true; +} + +void CodeGeneratorShared::verifyCompactNativeToBytecodeMap( + JitCode* code, const IonEntry::ScriptList& scripts, uint32_t numRegions) { +#ifdef DEBUG + MOZ_ASSERT(nativeToBytecodeMap_ != nullptr); + MOZ_ASSERT(nativeToBytecodeMapSize_ > 0); + MOZ_ASSERT(nativeToBytecodeTableOffset_ > 0); + MOZ_ASSERT(numRegions > 0); + + // The pointer to the table must be 4-byte aligned + const uint8_t* tablePtr = + nativeToBytecodeMap_.get() + nativeToBytecodeTableOffset_; + MOZ_ASSERT(uintptr_t(tablePtr) % sizeof(uint32_t) == 0); + + // Verify that numRegions was encoded correctly. + const JitcodeIonTable* ionTable = + reinterpret_cast<const JitcodeIonTable*>(tablePtr); + MOZ_ASSERT(ionTable->numRegions() == numRegions); + + // Region offset for first region should be at the start of the payload + // region. Since the offsets are backward from the start of the table, the + // first entry backoffset should be equal to the forward table offset from the + // start of the allocated data. + MOZ_ASSERT(ionTable->regionOffset(0) == nativeToBytecodeTableOffset_); + + // Verify each region. + for (uint32_t i = 0; i < ionTable->numRegions(); i++) { + // Back-offset must point into the payload region preceding the table, not + // before it. + MOZ_ASSERT(ionTable->regionOffset(i) <= nativeToBytecodeTableOffset_); + + // Back-offset must point to a later area in the payload region than + // previous back-offset. This means that back-offsets decrease + // monotonically. + MOZ_ASSERT_IF(i > 0, + ionTable->regionOffset(i) < ionTable->regionOffset(i - 1)); + + JitcodeRegionEntry entry = ionTable->regionEntry(i); + + // Ensure native code offset for region falls within jitcode. + MOZ_ASSERT(entry.nativeOffset() <= code->instructionsSize()); + + // Read out script/pc stack and verify. + JitcodeRegionEntry::ScriptPcIterator scriptPcIter = + entry.scriptPcIterator(); + while (scriptPcIter.hasMore()) { + uint32_t scriptIdx = 0, pcOffset = 0; + scriptPcIter.readNext(&scriptIdx, &pcOffset); + + // Ensure scriptIdx refers to a valid script in the list. + JSScript* script = scripts[scriptIdx].script; + + // Ensure pcOffset falls within the script. + MOZ_ASSERT(pcOffset < script->length()); + } + + // Obtain the original nativeOffset and pcOffset and script. + uint32_t curNativeOffset = entry.nativeOffset(); + JSScript* script = nullptr; + uint32_t curPcOffset = 0; + { + uint32_t scriptIdx = 0; + scriptPcIter.reset(); + scriptPcIter.readNext(&scriptIdx, &curPcOffset); + script = scripts[scriptIdx].script; + } + + // Read out nativeDeltas and pcDeltas and verify. + JitcodeRegionEntry::DeltaIterator deltaIter = entry.deltaIterator(); + while (deltaIter.hasMore()) { + uint32_t nativeDelta = 0; + int32_t pcDelta = 0; + deltaIter.readNext(&nativeDelta, &pcDelta); + + curNativeOffset += nativeDelta; + curPcOffset = uint32_t(int32_t(curPcOffset) + pcDelta); + + // Ensure that nativeOffset still falls within jitcode after delta. + MOZ_ASSERT(curNativeOffset <= code->instructionsSize()); + + // Ensure that pcOffset still falls within bytecode after delta. + MOZ_ASSERT(curPcOffset < script->length()); + } + } +#endif // DEBUG +} + +void CodeGeneratorShared::markSafepoint(LInstruction* ins) { + markSafepointAt(masm.currentOffset(), ins); +} + +void CodeGeneratorShared::markSafepointAt(uint32_t offset, LInstruction* ins) { + MOZ_ASSERT_IF( + !safepointIndices_.empty() && !masm.oom(), + offset - safepointIndices_.back().displacement() >= sizeof(uint32_t)); + masm.propagateOOM(safepointIndices_.append( + CodegenSafepointIndex(offset, ins->safepoint()))); +} + +void CodeGeneratorShared::ensureOsiSpace() { + // For a refresher, an invalidation point is of the form: + // 1: call <target> + // 2: ... + // 3: <osipoint> + // + // The four bytes *before* instruction 2 are overwritten with an offset. + // Callers must ensure that the instruction itself has enough bytes to + // support this. + // + // The bytes *at* instruction 3 are overwritten with an invalidation jump. + // jump. These bytes may be in a completely different IR sequence, but + // represent the join point of the call out of the function. + // + // At points where we want to ensure that invalidation won't corrupt an + // important instruction, we make sure to pad with nops. + if (masm.currentOffset() - lastOsiPointOffset_ < + Assembler::PatchWrite_NearCallSize()) { + int32_t paddingSize = Assembler::PatchWrite_NearCallSize(); + paddingSize -= masm.currentOffset() - lastOsiPointOffset_; + for (int32_t i = 0; i < paddingSize; ++i) { + masm.nop(); + } + } + MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() - lastOsiPointOffset_ >= + Assembler::PatchWrite_NearCallSize()); +} + +uint32_t CodeGeneratorShared::markOsiPoint(LOsiPoint* ins) { + encode(ins->snapshot()); + ensureOsiSpace(); + + uint32_t offset = masm.currentOffset(); + SnapshotOffset so = ins->snapshot()->snapshotOffset(); + masm.propagateOOM(osiIndices_.append(OsiIndex(offset, so))); + lastOsiPointOffset_ = offset; + + return offset; +} + +class OutOfLineTruncateSlow : public OutOfLineCodeBase<CodeGeneratorShared> { + FloatRegister src_; + Register dest_; + bool widenFloatToDouble_; + wasm::BytecodeOffset bytecodeOffset_; + bool preserveInstance_; + + public: + OutOfLineTruncateSlow( + FloatRegister src, Register dest, bool widenFloatToDouble = false, + wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset(), + bool preserveInstance = false) + : src_(src), + dest_(dest), + widenFloatToDouble_(widenFloatToDouble), + bytecodeOffset_(bytecodeOffset), + preserveInstance_(preserveInstance) {} + + void accept(CodeGeneratorShared* codegen) override { + codegen->visitOutOfLineTruncateSlow(this); + } + FloatRegister src() const { return src_; } + Register dest() const { return dest_; } + bool widenFloatToDouble() const { return widenFloatToDouble_; } + bool preserveInstance() const { return preserveInstance_; } + wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; } +}; + +OutOfLineCode* CodeGeneratorShared::oolTruncateDouble( + FloatRegister src, Register dest, MInstruction* mir, + wasm::BytecodeOffset bytecodeOffset, bool preserveInstance) { + MOZ_ASSERT_IF(IsCompilingWasm(), bytecodeOffset.isValid()); + + OutOfLineTruncateSlow* ool = new (alloc()) OutOfLineTruncateSlow( + src, dest, /* float32 */ false, bytecodeOffset, preserveInstance); + addOutOfLineCode(ool, mir); + return ool; +} + +void CodeGeneratorShared::emitTruncateDouble(FloatRegister src, Register dest, + MInstruction* mir) { + MOZ_ASSERT(mir->isTruncateToInt32() || mir->isWasmBuiltinTruncateToInt32()); + wasm::BytecodeOffset bytecodeOffset = + mir->isTruncateToInt32() + ? mir->toTruncateToInt32()->bytecodeOffset() + : mir->toWasmBuiltinTruncateToInt32()->bytecodeOffset(); + OutOfLineCode* ool = oolTruncateDouble(src, dest, mir, bytecodeOffset); + + masm.branchTruncateDoubleMaybeModUint32(src, dest, ool->entry()); + masm.bind(ool->rejoin()); +} + +void CodeGeneratorShared::emitTruncateFloat32(FloatRegister src, Register dest, + MInstruction* mir) { + MOZ_ASSERT(mir->isTruncateToInt32() || mir->isWasmBuiltinTruncateToInt32()); + wasm::BytecodeOffset bytecodeOffset = + mir->isTruncateToInt32() + ? mir->toTruncateToInt32()->bytecodeOffset() + : mir->toWasmBuiltinTruncateToInt32()->bytecodeOffset(); + OutOfLineTruncateSlow* ool = new (alloc()) + OutOfLineTruncateSlow(src, dest, /* float32 */ true, bytecodeOffset); + addOutOfLineCode(ool, mir); + + masm.branchTruncateFloat32MaybeModUint32(src, dest, ool->entry()); + masm.bind(ool->rejoin()); +} + +void CodeGeneratorShared::visitOutOfLineTruncateSlow( + OutOfLineTruncateSlow* ool) { + FloatRegister src = ool->src(); + Register dest = ool->dest(); + + saveVolatile(dest); + masm.outOfLineTruncateSlow(src, dest, ool->widenFloatToDouble(), + gen->compilingWasm(), ool->bytecodeOffset()); + restoreVolatile(dest); + + masm.jump(ool->rejoin()); +} + +bool CodeGeneratorShared::omitOverRecursedCheck() const { + // If the current function makes no calls (which means it isn't recursive) + // and it uses only a small amount of stack space, it doesn't need a + // stack overflow check. Note that the actual number here is somewhat + // arbitrary, and codegen actually uses small bounded amounts of + // additional stack space in some cases too. + return frameSize() < MAX_UNCHECKED_LEAF_FRAME_SIZE && + !gen->needsOverrecursedCheck(); +} + +void CodeGeneratorShared::emitPreBarrier(Register elements, + const LAllocation* index) { + if (index->isConstant()) { + Address address(elements, ToInt32(index) * sizeof(Value)); + masm.guardedCallPreBarrier(address, MIRType::Value); + } else { + BaseObjectElementIndex address(elements, ToRegister(index)); + masm.guardedCallPreBarrier(address, MIRType::Value); + } +} + +void CodeGeneratorShared::emitPreBarrier(Address address) { + masm.guardedCallPreBarrier(address, MIRType::Value); +} + +void CodeGeneratorShared::jumpToBlock(MBasicBlock* mir) { + // Skip past trivial blocks. + mir = skipTrivialBlocks(mir); + + // No jump necessary if we can fall through to the next block. + if (isNextBlock(mir->lir())) { + return; + } + + masm.jump(mir->lir()->label()); +} + +Label* CodeGeneratorShared::getJumpLabelForBranch(MBasicBlock* block) { + // Skip past trivial blocks. + return skipTrivialBlocks(block)->lir()->label(); +} + +// This function is not used for MIPS/MIPS64/LOONG64. They have +// branchToBlock. +#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) && \ + !defined(JS_CODEGEN_LOONG64) && !defined(JS_CODEGEN_RISCV64) +void CodeGeneratorShared::jumpToBlock(MBasicBlock* mir, + Assembler::Condition cond) { + // Skip past trivial blocks. + masm.j(cond, skipTrivialBlocks(mir)->lir()->label()); +} +#endif + +} // namespace jit +} // namespace js diff --git a/js/src/jit/shared/CodeGenerator-shared.h b/js/src/jit/shared/CodeGenerator-shared.h new file mode 100644 index 0000000000..66b0aff5af --- /dev/null +++ b/js/src/jit/shared/CodeGenerator-shared.h @@ -0,0 +1,488 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_CodeGenerator_shared_h +#define jit_shared_CodeGenerator_shared_h + +#include "mozilla/Alignment.h" + +#include <utility> + +#include "jit/InlineScriptTree.h" +#include "jit/JitcodeMap.h" +#include "jit/LIR.h" +#include "jit/MacroAssembler.h" +#include "jit/MIRGenerator.h" +#include "jit/MIRGraph.h" +#include "jit/SafepointIndex.h" +#include "jit/Safepoints.h" +#include "jit/Snapshots.h" + +namespace js { +namespace jit { + +class OutOfLineCode; +class CodeGenerator; +class MacroAssembler; +class IonIC; + +class OutOfLineTruncateSlow; + +class CodeGeneratorShared : public LElementVisitor { + js::Vector<OutOfLineCode*, 0, SystemAllocPolicy> outOfLineCode_; + + MacroAssembler& ensureMasm(MacroAssembler* masm, TempAllocator& alloc, + CompileRealm* realm); + mozilla::Maybe<IonHeapMacroAssembler> maybeMasm_; + + public: + MacroAssembler& masm; + + protected: + MIRGenerator* gen; + LIRGraph& graph; + LBlock* current; + SnapshotWriter snapshots_; + RecoverWriter recovers_; +#ifdef DEBUG + uint32_t pushedArgs_; +#endif + uint32_t lastOsiPointOffset_; + SafepointWriter safepoints_; + Label invalidate_; + CodeOffset invalidateEpilogueData_; + + // Label for the common return path. + NonAssertingLabel returnLabel_; + + js::Vector<CodegenSafepointIndex, 0, SystemAllocPolicy> safepointIndices_; + js::Vector<OsiIndex, 0, SystemAllocPolicy> osiIndices_; + + // Allocated data space needed at runtime. + js::Vector<uint8_t, 0, SystemAllocPolicy> runtimeData_; + + // Vector mapping each IC index to its offset in runtimeData_. + js::Vector<uint32_t, 0, SystemAllocPolicy> icList_; + + // IC data we need at compile-time. Discarded after creating the IonScript. + struct CompileTimeICInfo { + CodeOffset icOffsetForJump; + CodeOffset icOffsetForPush; + }; + js::Vector<CompileTimeICInfo, 0, SystemAllocPolicy> icInfo_; + + protected: + js::Vector<NativeToBytecode, 0, SystemAllocPolicy> nativeToBytecodeList_; + UniquePtr<uint8_t> nativeToBytecodeMap_; + uint32_t nativeToBytecodeMapSize_; + uint32_t nativeToBytecodeTableOffset_; + + bool isProfilerInstrumentationEnabled() { + return gen->isProfilerInstrumentationEnabled(); + } + + gc::Heap initialStringHeap() const { return gen->initialStringHeap(); } + gc::Heap initialBigIntHeap() const { return gen->initialBigIntHeap(); } + + protected: + // The offset of the first instruction of the OSR entry block from the + // beginning of the code buffer. + mozilla::Maybe<size_t> osrEntryOffset_ = {}; + + TempAllocator& alloc() const { return graph.mir().alloc(); } + + void setOsrEntryOffset(size_t offset) { osrEntryOffset_.emplace(offset); } + + size_t getOsrEntryOffset() const { + MOZ_RELEASE_ASSERT(osrEntryOffset_.isSome()); + return *osrEntryOffset_; + } + + typedef js::Vector<CodegenSafepointIndex, 8, SystemAllocPolicy> + SafepointIndices; + + protected: +#ifdef CHECK_OSIPOINT_REGISTERS + // See JitOptions.checkOsiPointRegisters. We set this here to avoid + // races when enableOsiPointRegisterChecks is called while we're generating + // code off-thread. + bool checkOsiPointRegisters; +#endif + + // The initial size of the frame in bytes. These are bytes beyond the + // constant header present for every Ion frame, used for pre-determined + // spills. + uint32_t frameDepth_; + + // Offset in bytes to the incoming arguments, relative to the frame pointer. + uint32_t offsetOfArgsFromFP_ = 0; + + // Offset in bytes of the stack region reserved for passed argument Values. + uint32_t offsetOfPassedArgSlots_ = 0; + + // For argument construction for calls. Argslots are Value-sized. + inline Address AddressOfPassedArg(uint32_t slot) const; + inline uint32_t UnusedStackBytesForCall(uint32_t numArgSlots) const; + + template <BaseRegForAddress Base = BaseRegForAddress::Default> + inline Address ToAddress(const LAllocation& a) const; + + template <BaseRegForAddress Base = BaseRegForAddress::Default> + inline Address ToAddress(const LAllocation* a) const; + + static inline Address ToAddress(Register elements, const LAllocation* index, + Scalar::Type type, + int32_t offsetAdjustment = 0); + + uint32_t frameSize() const { return frameDepth_; } + + protected: + bool addNativeToBytecodeEntry(const BytecodeSite* site); + void dumpNativeToBytecodeEntries(); + void dumpNativeToBytecodeEntry(uint32_t idx); + + public: + MIRGenerator& mirGen() const { return *gen; } + + // When appending to runtimeData_, the vector might realloc, leaving pointers + // int the origianl vector stale and unusable. DataPtr acts like a pointer, + // but allows safety in the face of potentially realloc'ing vector appends. + friend class DataPtr; + template <typename T> + class DataPtr { + CodeGeneratorShared* cg_; + size_t index_; + + T* lookup() { return reinterpret_cast<T*>(&cg_->runtimeData_[index_]); } + + public: + DataPtr(CodeGeneratorShared* cg, size_t index) : cg_(cg), index_(index) {} + + T* operator->() { return lookup(); } + T* operator*() { return lookup(); } + }; + + protected: + [[nodiscard]] bool allocateData(size_t size, size_t* offset) { + MOZ_ASSERT(size % sizeof(void*) == 0); + *offset = runtimeData_.length(); + masm.propagateOOM(runtimeData_.appendN(0, size)); + return !masm.oom(); + } + + template <typename T> + inline size_t allocateIC(const T& cache) { + static_assert(std::is_base_of_v<IonIC, T>, "T must inherit from IonIC"); + size_t index; + masm.propagateOOM( + allocateData(sizeof(mozilla::AlignedStorage2<T>), &index)); + masm.propagateOOM(icList_.append(index)); + masm.propagateOOM(icInfo_.append(CompileTimeICInfo())); + if (masm.oom()) { + return SIZE_MAX; + } + // Use the copy constructor on the allocated space. + MOZ_ASSERT(index == icList_.back()); + new (&runtimeData_[index]) T(cache); + return index; + } + + protected: + // Encodes an LSnapshot into the compressed snapshot buffer. + void encode(LRecoverInfo* recover); + void encode(LSnapshot* snapshot); + void encodeAllocation(LSnapshot* snapshot, MDefinition* def, + uint32_t* startIndex); + + // Encode all encountered safepoints in CG-order, and resolve |indices| for + // safepoint offsets. + bool encodeSafepoints(); + + // Fixup offsets of native-to-bytecode map. + bool createNativeToBytecodeScriptList(JSContext* cx, + IonEntry::ScriptList& scripts); + bool generateCompactNativeToBytecodeMap(JSContext* cx, JitCode* code, + IonEntry::ScriptList& scripts); + void verifyCompactNativeToBytecodeMap(JitCode* code, + const IonEntry::ScriptList& scripts, + uint32_t numRegions); + + // Mark the safepoint on |ins| as corresponding to the current assembler + // location. The location should be just after a call. + void markSafepoint(LInstruction* ins); + void markSafepointAt(uint32_t offset, LInstruction* ins); + + // Mark the OSI point |ins| as corresponding to the current + // assembler location inside the |osiIndices_|. Return the assembler + // location for the OSI point return location. + uint32_t markOsiPoint(LOsiPoint* ins); + + // Ensure that there is enough room between the last OSI point and the + // current instruction, such that: + // (1) Invalidation will not overwrite the current instruction, and + // (2) Overwriting the current instruction will not overwrite + // an invalidation marker. + void ensureOsiSpace(); + + OutOfLineCode* oolTruncateDouble( + FloatRegister src, Register dest, MInstruction* mir, + wasm::BytecodeOffset callOffset = wasm::BytecodeOffset(), + bool preserveInstance = false); + void emitTruncateDouble(FloatRegister src, Register dest, MInstruction* mir); + void emitTruncateFloat32(FloatRegister src, Register dest, MInstruction* mir); + + void emitPreBarrier(Register elements, const LAllocation* index); + void emitPreBarrier(Address address); + + // We don't emit code for trivial blocks, so if we want to branch to the + // given block, and it's trivial, return the ultimate block we should + // actually branch directly to. + MBasicBlock* skipTrivialBlocks(MBasicBlock* block) { + while (block->lir()->isTrivial()) { + LGoto* ins = block->lir()->rbegin()->toGoto(); + MOZ_ASSERT(ins->numSuccessors() == 1); + block = ins->getSuccessor(0); + } + return block; + } + + // Test whether the given block can be reached via fallthrough from the + // current block. + inline bool isNextBlock(LBlock* block) { + uint32_t target = skipTrivialBlocks(block->mir())->id(); + uint32_t i = current->mir()->id() + 1; + if (target < i) { + return false; + } + // Trivial blocks can be crossed via fallthrough. + for (; i != target; ++i) { + if (!graph.getBlock(i)->isTrivial()) { + return false; + } + } + return true; + } + + protected: + // Save and restore all volatile registers to/from the stack, excluding the + // specified register(s), before a function call made using callWithABI and + // after storing the function call's return value to an output register. + // (The only registers that don't need to be saved/restored are 1) the + // temporary register used to store the return value of the function call, + // if there is one [otherwise that stored value would be overwritten]; and + // 2) temporary registers whose values aren't needed in the rest of the LIR + // instruction [this is purely an optimization]. All other volatiles must + // be saved and restored in case future LIR instructions need those values.) + void saveVolatile(Register output) { + LiveRegisterSet regs(RegisterSet::Volatile()); + regs.takeUnchecked(output); + masm.PushRegsInMask(regs); + } + void restoreVolatile(Register output) { + LiveRegisterSet regs(RegisterSet::Volatile()); + regs.takeUnchecked(output); + masm.PopRegsInMask(regs); + } + void saveVolatile(FloatRegister output) { + LiveRegisterSet regs(RegisterSet::Volatile()); + regs.takeUnchecked(output); + masm.PushRegsInMask(regs); + } + void restoreVolatile(FloatRegister output) { + LiveRegisterSet regs(RegisterSet::Volatile()); + regs.takeUnchecked(output); + masm.PopRegsInMask(regs); + } + void saveVolatile(LiveRegisterSet temps) { + masm.PushRegsInMask(LiveRegisterSet(RegisterSet::VolatileNot(temps.set()))); + } + void restoreVolatile(LiveRegisterSet temps) { + masm.PopRegsInMask(LiveRegisterSet(RegisterSet::VolatileNot(temps.set()))); + } + void saveVolatile() { + masm.PushRegsInMask(LiveRegisterSet(RegisterSet::Volatile())); + } + void restoreVolatile() { + masm.PopRegsInMask(LiveRegisterSet(RegisterSet::Volatile())); + } + + // These functions have to be called before and after any callVM and before + // any modifications of the stack. Modification of the stack made after + // these calls should update the framePushed variable, needed by the exit + // frame produced by callVM. + inline void saveLive(LInstruction* ins); + inline void restoreLive(LInstruction* ins); + inline void restoreLiveIgnore(LInstruction* ins, LiveRegisterSet reg); + + // Get/save/restore all registers that are both live and volatile. + inline LiveRegisterSet liveVolatileRegs(LInstruction* ins); + inline void saveLiveVolatile(LInstruction* ins); + inline void restoreLiveVolatile(LInstruction* ins); + + public: + template <typename T> + void pushArg(const T& t) { + masm.Push(t); +#ifdef DEBUG + pushedArgs_++; +#endif + } + + void pushArg(jsid id, Register temp) { + masm.Push(id, temp); +#ifdef DEBUG + pushedArgs_++; +#endif + } + + template <typename T> + CodeOffset pushArgWithPatch(const T& t) { +#ifdef DEBUG + pushedArgs_++; +#endif + return masm.PushWithPatch(t); + } + + void storePointerResultTo(Register reg) { masm.storeCallPointerResult(reg); } + + void storeFloatResultTo(FloatRegister reg) { masm.storeCallFloatResult(reg); } + + template <typename T> + void storeResultValueTo(const T& t) { + masm.storeCallResultValue(t); + } + + protected: + void addIC(LInstruction* lir, size_t cacheIndex); + + protected: + bool generatePrologue(); + bool generateEpilogue(); + + void addOutOfLineCode(OutOfLineCode* code, const MInstruction* mir); + void addOutOfLineCode(OutOfLineCode* code, const BytecodeSite* site); + bool generateOutOfLineCode(); + + Label* getJumpLabelForBranch(MBasicBlock* block); + + // Generate a jump to the start of the specified block. Use this in place of + // jumping directly to mir->lir()->label(), or use getJumpLabelForBranch() + // if a label to use directly is needed. + void jumpToBlock(MBasicBlock* mir); + +// This function is not used for MIPS. MIPS has branchToBlock. +#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) + void jumpToBlock(MBasicBlock* mir, Assembler::Condition cond); +#endif + + private: + void generateInvalidateEpilogue(); + + public: + CodeGeneratorShared(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm); + + public: + void visitOutOfLineTruncateSlow(OutOfLineTruncateSlow* ool); + + bool omitOverRecursedCheck() const; + + public: + bool isGlobalObject(JSObject* object); +}; + +// An out-of-line path is generated at the end of the function. +class OutOfLineCode : public TempObject { + Label entry_; + Label rejoin_; + uint32_t framePushed_; + const BytecodeSite* site_; + + public: + OutOfLineCode() : framePushed_(0), site_() {} + + virtual void generate(CodeGeneratorShared* codegen) = 0; + + Label* entry() { return &entry_; } + virtual void bind(MacroAssembler* masm) { masm->bind(entry()); } + Label* rejoin() { return &rejoin_; } + void setFramePushed(uint32_t framePushed) { framePushed_ = framePushed; } + uint32_t framePushed() const { return framePushed_; } + void setBytecodeSite(const BytecodeSite* site) { site_ = site; } + const BytecodeSite* bytecodeSite() const { return site_; } +}; + +// For OOL paths that want a specific-typed code generator. +template <typename T> +class OutOfLineCodeBase : public OutOfLineCode { + public: + virtual void generate(CodeGeneratorShared* codegen) override { + accept(static_cast<T*>(codegen)); + } + + public: + virtual void accept(T* codegen) = 0; +}; + +template <class CodeGen> +class OutOfLineWasmTruncateCheckBase : public OutOfLineCodeBase<CodeGen> { + MIRType fromType_; + MIRType toType_; + FloatRegister input_; + Register output_; + Register64 output64_; + TruncFlags flags_; + wasm::BytecodeOffset bytecodeOffset_; + + public: + OutOfLineWasmTruncateCheckBase(MWasmTruncateToInt32* mir, FloatRegister input, + Register output) + : fromType_(mir->input()->type()), + toType_(MIRType::Int32), + input_(input), + output_(output), + output64_(Register64::Invalid()), + flags_(mir->flags()), + bytecodeOffset_(mir->bytecodeOffset()) {} + + OutOfLineWasmTruncateCheckBase(MWasmBuiltinTruncateToInt64* mir, + FloatRegister input, Register64 output) + : fromType_(mir->input()->type()), + toType_(MIRType::Int64), + input_(input), + output_(Register::Invalid()), + output64_(output), + flags_(mir->flags()), + bytecodeOffset_(mir->bytecodeOffset()) {} + + OutOfLineWasmTruncateCheckBase(MWasmTruncateToInt64* mir, FloatRegister input, + Register64 output) + : fromType_(mir->input()->type()), + toType_(MIRType::Int64), + input_(input), + output_(Register::Invalid()), + output64_(output), + flags_(mir->flags()), + bytecodeOffset_(mir->bytecodeOffset()) {} + + void accept(CodeGen* codegen) override { + codegen->visitOutOfLineWasmTruncateCheck(this); + } + + FloatRegister input() const { return input_; } + Register output() const { return output_; } + Register64 output64() const { return output64_; } + MIRType toType() const { return toType_; } + MIRType fromType() const { return fromType_; } + bool isUnsigned() const { return flags_ & TRUNC_UNSIGNED; } + bool isSaturating() const { return flags_ & TRUNC_SATURATING; } + TruncFlags flags() const { return flags_; } + wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_shared_CodeGenerator_shared_h */ diff --git a/js/src/jit/shared/Disassembler-shared.cpp b/js/src/jit/shared/Disassembler-shared.cpp new file mode 100644 index 0000000000..a8e7f126b5 --- /dev/null +++ b/js/src/jit/shared/Disassembler-shared.cpp @@ -0,0 +1,248 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/shared/Disassembler-shared.h" + +#include "jit/JitSpewer.h" +#include "jit/Label.h" +#include "js/Printer.h" + +using namespace js::jit; + +using js::Sprinter; + +#ifdef JS_DISASM_SUPPORTED +// Concurrent assemblers are disambiguated by prefixing every disassembly with a +// tag that is quasi-unique, and certainly unique enough in realistic cases +// where we are debugging and looking at disassembler output. The tag is a +// letter or digit between brackets prefixing the disassembly, eg, [X]. This +// wraps around every 62 assemblers. +// +// When running with --no-threads we can still have concurrent assemblers in the +// form of nested assemblers, as when an IC stub is created by one assembler +// while a JS compilation is going on and producing output in another assembler. +// +// We generate the tag for an assembler by incrementing a global mod-2^32 +// counter every time a new disassembler is created. + +mozilla::Atomic<uint32_t> DisassemblerSpew::counter_(0); +#endif + +DisassemblerSpew::DisassemblerSpew() + : printer_(nullptr) +#ifdef JS_DISASM_SUPPORTED + , + labelIndent_(""), + targetIndent_(""), + spewNext_(1000), + nodes_(nullptr), + tag_(0) +#endif +{ +#ifdef JS_DISASM_SUPPORTED + tag_ = counter_++; +#endif +} + +DisassemblerSpew::~DisassemblerSpew() { +#ifdef JS_DISASM_SUPPORTED + Node* p = nodes_; + while (p) { + Node* victim = p; + p = p->next; + js_free(victim); + } +#endif +} + +void DisassemblerSpew::setPrinter(Sprinter* printer) { printer_ = printer; } + +bool DisassemblerSpew::isDisabled() { + return !(JitSpewEnabled(JitSpew_Codegen) || printer_); +} + +void DisassemblerSpew::spew(const char* fmt, ...) { +#ifdef JS_DISASM_SUPPORTED + static const char prefix_chars[] = + "0123456789" + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + static const char prefix_fmt[] = "[%c] "; + + char fmt2[1024]; + if (sizeof(fmt2) >= strlen(fmt) + sizeof(prefix_fmt)) { + snprintf(fmt2, sizeof(prefix_fmt), prefix_fmt, + prefix_chars[tag_ % (sizeof(prefix_chars) - 1)]); + strcat(fmt2, fmt); + fmt = fmt2; + } +#endif + + va_list args; + va_start(args, fmt); + spewVA(fmt, args); + va_end(args); +} + +void DisassemblerSpew::spewVA(const char* fmt, va_list va) { + if (printer_) { + printer_->vprintf(fmt, va); + printer_->put("\n"); + } + js::jit::JitSpewVA(js::jit::JitSpew_Codegen, fmt, va); +} + +#ifdef JS_DISASM_SUPPORTED + +void DisassemblerSpew::setLabelIndent(const char* s) { labelIndent_ = s; } + +void DisassemblerSpew::setTargetIndent(const char* s) { targetIndent_ = s; } + +DisassemblerSpew::LabelDoc DisassemblerSpew::refLabel(const Label* l) { + return l ? LabelDoc(internalResolve(l), l->bound()) : LabelDoc(); +} + +void DisassemblerSpew::spewRef(const LabelDoc& target) { + if (isDisabled()) { + return; + } + if (!target.valid) { + return; + } + spew("%s-> %d%s", targetIndent_, target.doc, !target.bound ? "f" : ""); +} + +void DisassemblerSpew::spewBind(const Label* label) { + if (isDisabled()) { + return; + } + uint32_t v = internalResolve(label); + Node* probe = lookup(label); + if (probe) { + probe->bound = true; + } + spew("%s%d:", labelIndent_, v); +} + +void DisassemblerSpew::spewRetarget(const Label* label, const Label* target) { + if (isDisabled()) { + return; + } + LabelDoc labelDoc = LabelDoc(internalResolve(label), label->bound()); + LabelDoc targetDoc = LabelDoc(internalResolve(target), target->bound()); + Node* probe = lookup(label); + if (probe) { + probe->bound = true; + } + spew("%s%d: .retarget -> %d%s", labelIndent_, labelDoc.doc, targetDoc.doc, + !targetDoc.bound ? "f" : ""); +} + +void DisassemblerSpew::formatLiteral(const LiteralDoc& doc, char* buffer, + size_t bufsize) { + switch (doc.type) { + case LiteralDoc::Type::Patchable: + snprintf(buffer, bufsize, "patchable"); + break; + case LiteralDoc::Type::I32: + snprintf(buffer, bufsize, "%d", doc.value.i32); + break; + case LiteralDoc::Type::U32: + snprintf(buffer, bufsize, "%u", doc.value.u32); + break; + case LiteralDoc::Type::I64: + snprintf(buffer, bufsize, "%" PRIi64, doc.value.i64); + break; + case LiteralDoc::Type::U64: + snprintf(buffer, bufsize, "%" PRIu64, doc.value.u64); + break; + case LiteralDoc::Type::F32: + snprintf(buffer, bufsize, "%g", doc.value.f32); + break; + case LiteralDoc::Type::F64: + snprintf(buffer, bufsize, "%g", doc.value.f64); + break; + default: + MOZ_CRASH(); + } +} + +void DisassemblerSpew::spewOrphans() { + for (Node* p = nodes_; p; p = p->next) { + if (!p->bound) { + spew("%s%d: ; .orphan", labelIndent_, p->value); + } + } +} + +uint32_t DisassemblerSpew::internalResolve(const Label* l) { + // Note, internalResolve will sometimes return 0 when it is triggered by the + // profiler and not by a full disassembly, since in that case a label can be + // used or bound but not previously have been defined. In that case, + // internalResolve(l) will not necessarily create a binding for l! + // Consequently a subsequent lookup(l) may still return null. + return l->used() || l->bound() ? probe(l) : define(l); +} + +uint32_t DisassemblerSpew::probe(const Label* l) { + Node* n = lookup(l); + return n ? n->value : 0; +} + +uint32_t DisassemblerSpew::define(const Label* l) { + remove(l); + uint32_t value = spewNext_++; + if (!add(l, value)) { + return 0; + } + return value; +} + +DisassemblerSpew::Node* DisassemblerSpew::lookup(const Label* key) { + Node* p; + for (p = nodes_; p && p->key != key; p = p->next) { + ; + } + return p; +} + +DisassemblerSpew::Node* DisassemblerSpew::add(const Label* key, + uint32_t value) { + MOZ_ASSERT(!lookup(key)); + Node* node = js_new<Node>(); + if (node) { + node->key = key; + node->value = value; + node->bound = false; + node->next = nodes_; + nodes_ = node; + } + return node; +} + +bool DisassemblerSpew::remove(const Label* key) { + // We do not require that there is a node matching the key. + for (Node *p = nodes_, *pp = nullptr; p; pp = p, p = p->next) { + if (p->key == key) { + if (pp) { + pp->next = p->next; + } else { + nodes_ = p->next; + } + js_free(p); + return true; + } + } + return false; +} + +#else + +DisassemblerSpew::LabelDoc DisassemblerSpew::refLabel(const Label* l) { + return LabelDoc(); +} + +#endif diff --git a/js/src/jit/shared/Disassembler-shared.h b/js/src/jit/shared/Disassembler-shared.h new file mode 100644 index 0000000000..882de421a8 --- /dev/null +++ b/js/src/jit/shared/Disassembler-shared.h @@ -0,0 +1,184 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_Disassembler_shared_h +#define jit_shared_Disassembler_shared_h + +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" + +#include <stdarg.h> +#include <stddef.h> +#include <stdint.h> + +#include "jstypes.h" // JS_PUBLIC_API + +#if defined(JS_DISASM_ARM) || defined(JS_DISASM_ARM64) +# define JS_DISASM_SUPPORTED +#endif + +namespace js { + +class JS_PUBLIC_API Sprinter; + +namespace jit { + +class Label; + +// A wrapper around spew/disassembly functionality. The disassembler is built +// on a per-instruction disassembler (as in our ARM, ARM64 back-ends) and +// formats labels with meaningful names and literals with meaningful values, if +// the assembler creates documentation (with provided helpers) at appropriate +// points. + +class DisassemblerSpew { +#ifdef JS_DISASM_SUPPORTED + struct Node { + const Label* key; // Never dereferenced, only used for its value + uint32_t value; // The printable label value + bool bound; // If the label has been seen by spewBind() + Node* next; + }; + + Node* lookup(const Label* key); + Node* add(const Label* key, uint32_t value); + bool remove(const Label* key); + + uint32_t probe(const Label* l); + uint32_t define(const Label* l); + uint32_t internalResolve(const Label* l); +#endif + + void spewVA(const char* fmt, va_list args) MOZ_FORMAT_PRINTF(2, 0); + + public: + DisassemblerSpew(); + ~DisassemblerSpew(); + +#ifdef JS_DISASM_SUPPORTED + // Set indentation strings. The spewer retains a reference to s. + void setLabelIndent(const char* s); + void setTargetIndent(const char* s); +#endif + + // Set the spew printer, which will always be used if it is set, regardless + // of whether the system spew channel is enabled or not. The spewer retains + // a reference to sp. + void setPrinter(Sprinter* sp); + + // Return true if disassembly spew is disabled and no additional printer is + // set. + bool isDisabled(); + + // Format and print text on the spew channel; output is suppressed if spew + // is disabled. The output is not indented, and is terminated by a newline. + void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3); + + // Documentation for a label reference. + struct LabelDoc { +#ifdef JS_DISASM_SUPPORTED + LabelDoc() : doc(0), bound(false), valid(false) {} + LabelDoc(uint32_t doc, bool bound) : doc(doc), bound(bound), valid(true) {} + const uint32_t doc; + const bool bound; + const bool valid; +#else + LabelDoc() = default; + LabelDoc(uint32_t, bool) {} +#endif + }; + + // Documentation for a literal load. + struct LiteralDoc { +#ifdef JS_DISASM_SUPPORTED + enum class Type { Patchable, I32, U32, I64, U64, F32, F64 }; + const Type type; + union { + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + float f32; + double f64; + } value; + LiteralDoc() : type(Type::Patchable) {} + explicit LiteralDoc(int32_t v) : type(Type::I32) { value.i32 = v; } + explicit LiteralDoc(uint32_t v) : type(Type::U32) { value.u32 = v; } + explicit LiteralDoc(int64_t v) : type(Type::I64) { value.i64 = v; } + explicit LiteralDoc(uint64_t v) : type(Type::U64) { value.u64 = v; } + explicit LiteralDoc(float v) : type(Type::F32) { value.f32 = v; } + explicit LiteralDoc(double v) : type(Type::F64) { value.f64 = v; } +#else + LiteralDoc() = default; + explicit LiteralDoc(int32_t) {} + explicit LiteralDoc(uint32_t) {} + explicit LiteralDoc(int64_t) {} + explicit LiteralDoc(uint64_t) {} + explicit LiteralDoc(float) {} + explicit LiteralDoc(double) {} +#endif + }; + + // Reference a label, resolving it to a printable representation. + // + // NOTE: The printable representation depends on the state of the label, so + // if we call resolve() when emitting & disassembling a branch instruction + // then it should be called before the label becomes Used, if emitting the + // branch can change the label's state. + // + // If the disassembler is not defined this returns a structure that is + // marked not valid. + LabelDoc refLabel(const Label* l); + +#ifdef JS_DISASM_SUPPORTED + // Spew the label information previously gathered by refLabel(), at a point + // where the label is referenced. The output is indented by targetIndent_ + // and terminated by a newline. + void spewRef(const LabelDoc& target); + + // Spew the label at the point where the label is bound. The output is + // indented by labelIndent_ and terminated by a newline. + void spewBind(const Label* label); + + // Spew a retarget directive at the point where the retarget is recorded. + // The output is indented by labelIndent_ and terminated by a newline. + void spewRetarget(const Label* label, const Label* target); + + // Format a literal value into the buffer. The buffer is always + // NUL-terminated even if this chops the formatted value. + void formatLiteral(const LiteralDoc& doc, char* buffer, size_t bufsize); + + // Print any unbound labels, one per line, with normal label indent and with + // a comment indicating the label is not defined. Labels can be referenced + // but unbound in some legitimate cases, normally for traps. Printing them + // reduces confusion. + void spewOrphans(); +#endif + + private: + Sprinter* printer_; +#ifdef JS_DISASM_SUPPORTED + const char* labelIndent_; + const char* targetIndent_; + uint32_t spewNext_; + Node* nodes_; + uint32_t tag_; + + // This global is used to disambiguate concurrently live assemblers, see + // comments in Disassembler-shared.cpp for why this is desirable. + // + // The variable is atomic to avoid any kind of complaint from thread + // sanitizers etc. However, trying to look at disassembly without using + // --no-threads is basically insane, so you can ignore the multi-threading + // implications here. + static mozilla::Atomic<uint32_t> counter_; +#endif +}; + +} // namespace jit +} // namespace js + +#endif // jit_shared_Disassembler_shared_h diff --git a/js/src/jit/shared/IonAssemblerBuffer.h b/js/src/jit/shared/IonAssemblerBuffer.h new file mode 100644 index 0000000000..170f098707 --- /dev/null +++ b/js/src/jit/shared/IonAssemblerBuffer.h @@ -0,0 +1,438 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_IonAssemblerBuffer_h +#define jit_shared_IonAssemblerBuffer_h + +#include "mozilla/Assertions.h" +#include "mozilla/MathAlgorithms.h" + +#include <algorithm> + +#include "jit/ProcessExecutableMemory.h" +#include "jit/shared/Assembler-shared.h" + +namespace js { +namespace jit { + +// The offset into a buffer, in bytes. +class BufferOffset { + int offset; + + public: + friend BufferOffset nextOffset(); + + BufferOffset() : offset(INT_MIN) {} + + explicit BufferOffset(int offset_) : offset(offset_) { + MOZ_ASSERT(offset >= 0); + } + + explicit BufferOffset(Label* l) : offset(l->offset()) { + MOZ_ASSERT(offset >= 0); + } + + int getOffset() const { return offset; } + bool assigned() const { return offset != INT_MIN; } + + // A BOffImm is a Branch Offset Immediate. It is an architecture-specific + // structure that holds the immediate for a pc relative branch. diffB takes + // the label for the destination of the branch, and encodes the immediate + // for the branch. This will need to be fixed up later, since A pool may be + // inserted between the branch and its destination. + template <class BOffImm> + BOffImm diffB(BufferOffset other) const { + if (!BOffImm::IsInRange(offset - other.offset)) { + return BOffImm(); + } + return BOffImm(offset - other.offset); + } + + template <class BOffImm> + BOffImm diffB(Label* other) const { + MOZ_ASSERT(other->bound()); + if (!BOffImm::IsInRange(offset - other->offset())) { + return BOffImm(); + } + return BOffImm(offset - other->offset()); + } +}; + +inline bool operator<(BufferOffset a, BufferOffset b) { + return a.getOffset() < b.getOffset(); +} + +inline bool operator>(BufferOffset a, BufferOffset b) { + return a.getOffset() > b.getOffset(); +} + +inline bool operator<=(BufferOffset a, BufferOffset b) { + return a.getOffset() <= b.getOffset(); +} + +inline bool operator>=(BufferOffset a, BufferOffset b) { + return a.getOffset() >= b.getOffset(); +} + +inline bool operator==(BufferOffset a, BufferOffset b) { + return a.getOffset() == b.getOffset(); +} + +inline bool operator!=(BufferOffset a, BufferOffset b) { + return a.getOffset() != b.getOffset(); +} + +template <int SliceSize> +class BufferSlice { + protected: + BufferSlice<SliceSize>* prev_; + BufferSlice<SliceSize>* next_; + + size_t bytelength_; + + public: + mozilla::Array<uint8_t, SliceSize> instructions; + + public: + explicit BufferSlice() : prev_(nullptr), next_(nullptr), bytelength_(0) {} + + size_t length() const { return bytelength_; } + static inline size_t Capacity() { return SliceSize; } + + BufferSlice* getNext() const { return next_; } + BufferSlice* getPrev() const { return prev_; } + + void setNext(BufferSlice<SliceSize>* next) { + MOZ_ASSERT(next_ == nullptr); + MOZ_ASSERT(next->prev_ == nullptr); + next_ = next; + next->prev_ = this; + } + + void putBytes(size_t numBytes, const void* source) { + MOZ_ASSERT(bytelength_ + numBytes <= SliceSize); + if (source) { + memcpy(&instructions[length()], source, numBytes); + } + bytelength_ += numBytes; + } + + MOZ_ALWAYS_INLINE + void putU32Aligned(uint32_t value) { + MOZ_ASSERT(bytelength_ + 4 <= SliceSize); + MOZ_ASSERT((bytelength_ & 3) == 0); + MOZ_ASSERT((uintptr_t(&instructions[0]) & 3) == 0); + *reinterpret_cast<uint32_t*>(&instructions[bytelength_]) = value; + bytelength_ += 4; + } +}; + +template <int SliceSize, class Inst> +class AssemblerBuffer { + protected: + typedef BufferSlice<SliceSize> Slice; + + // Doubly-linked list of BufferSlices, with the most recent in tail position. + Slice* head; + Slice* tail; + + bool m_oom; + + // How many bytes has been committed to the buffer thus far. + // Does not include tail. + uint32_t bufferSize; + + // How many bytes can be in the buffer. Normally this is + // MaxCodeBytesPerBuffer, but for pasteup buffers where we handle far jumps + // explicitly it can be larger. + uint32_t maxSize; + + // Finger for speeding up accesses. + Slice* finger; + int finger_offset; + + LifoAlloc lifoAlloc_; + + public: + explicit AssemblerBuffer() + : head(nullptr), + tail(nullptr), + m_oom(false), + bufferSize(0), + maxSize(MaxCodeBytesPerBuffer), + finger(nullptr), + finger_offset(0), + lifoAlloc_(8192) {} + + public: + bool isAligned(size_t alignment) const { + MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment)); + return !(size() & (alignment - 1)); + } + + void setUnlimited() { maxSize = MaxCodeBytesPerProcess; } + + private: + Slice* newSlice(LifoAlloc& a) { + if (size() > maxSize - sizeof(Slice)) { + fail_oom(); + return nullptr; + } + Slice* tmp = static_cast<Slice*>(a.alloc(sizeof(Slice))); + if (!tmp) { + fail_oom(); + return nullptr; + } + return new (tmp) Slice; + } + + public: + bool ensureSpace(size_t size) { + // Space can exist in the most recent Slice. + if (tail && tail->length() + size <= tail->Capacity()) { + // Simulate allocation failure even when we don't need a new slice. + if (js::oom::ShouldFailWithOOM()) { + return fail_oom(); + } + + return true; + } + + // Otherwise, a new Slice must be added. + Slice* slice = newSlice(lifoAlloc_); + if (slice == nullptr) { + return fail_oom(); + } + + // If this is the first Slice in the buffer, add to head position. + if (!head) { + head = slice; + finger = slice; + finger_offset = 0; + } + + // Finish the last Slice and add the new Slice to the linked list. + if (tail) { + bufferSize += tail->length(); + tail->setNext(slice); + } + tail = slice; + + return true; + } + + BufferOffset putByte(uint8_t value) { + return putBytes(sizeof(value), &value); + } + + BufferOffset putShort(uint16_t value) { + return putBytes(sizeof(value), &value); + } + + BufferOffset putInt(uint32_t value) { + return putBytes(sizeof(value), &value); + } + + MOZ_ALWAYS_INLINE + BufferOffset putU32Aligned(uint32_t value) { + if (!ensureSpace(sizeof(value))) { + return BufferOffset(); + } + + BufferOffset ret = nextOffset(); + tail->putU32Aligned(value); + return ret; + } + + // Add numBytes bytes to this buffer. + // The data must fit in a single slice. + BufferOffset putBytes(size_t numBytes, const void* inst) { + if (!ensureSpace(numBytes)) { + return BufferOffset(); + } + + BufferOffset ret = nextOffset(); + tail->putBytes(numBytes, inst); + return ret; + } + + // Add a potentially large amount of data to this buffer. + // The data may be distrubuted across multiple slices. + // Return the buffer offset of the first added byte. + BufferOffset putBytesLarge(size_t numBytes, const void* data) { + BufferOffset ret = nextOffset(); + while (numBytes > 0) { + if (!ensureSpace(1)) { + return BufferOffset(); + } + size_t avail = tail->Capacity() - tail->length(); + size_t xfer = numBytes < avail ? numBytes : avail; + MOZ_ASSERT(xfer > 0, "ensureSpace should have allocated a slice"); + tail->putBytes(xfer, data); + data = (const uint8_t*)data + xfer; + numBytes -= xfer; + } + return ret; + } + + unsigned int size() const { + if (tail) { + return bufferSize + tail->length(); + } + return bufferSize; + } + BufferOffset nextOffset() const { return BufferOffset(size()); } + + bool oom() const { return m_oom; } + + bool fail_oom() { + m_oom = true; +#ifdef DEBUG + JitContext* context = MaybeGetJitContext(); + if (context) { + context->setOOM(); + } +#endif + return false; + } + + private: + void update_finger(Slice* finger_, int fingerOffset_) { + finger = finger_; + finger_offset = fingerOffset_; + } + + static const unsigned SliceDistanceRequiringFingerUpdate = 3; + + Inst* getInstForwards(BufferOffset off, Slice* start, int startOffset, + bool updateFinger = false) { + const int offset = off.getOffset(); + + int cursor = startOffset; + unsigned slicesSkipped = 0; + + MOZ_ASSERT(offset >= cursor); + + for (Slice* slice = start; slice != nullptr; slice = slice->getNext()) { + const int slicelen = slice->length(); + + // Is the offset within the bounds of this slice? + if (offset < cursor + slicelen) { + if (updateFinger || + slicesSkipped >= SliceDistanceRequiringFingerUpdate) { + update_finger(slice, cursor); + } + + MOZ_ASSERT(offset - cursor < (int)slice->length()); + return (Inst*)&slice->instructions[offset - cursor]; + } + + cursor += slicelen; + slicesSkipped++; + } + + MOZ_CRASH("Invalid instruction cursor."); + } + + Inst* getInstBackwards(BufferOffset off, Slice* start, int startOffset, + bool updateFinger = false) { + const int offset = off.getOffset(); + + int cursor = startOffset; // First (lowest) offset in the start Slice. + unsigned slicesSkipped = 0; + + MOZ_ASSERT(offset < int(cursor + start->length())); + + for (Slice* slice = start; slice != nullptr;) { + // Is the offset within the bounds of this slice? + if (offset >= cursor) { + if (updateFinger || + slicesSkipped >= SliceDistanceRequiringFingerUpdate) { + update_finger(slice, cursor); + } + + MOZ_ASSERT(offset - cursor < (int)slice->length()); + return (Inst*)&slice->instructions[offset - cursor]; + } + + // Move the cursor to the start of the previous slice. + Slice* prev = slice->getPrev(); + cursor -= prev->length(); + + slice = prev; + slicesSkipped++; + } + + MOZ_CRASH("Invalid instruction cursor."); + } + + public: + Inst* getInstOrNull(BufferOffset off) { + if (!off.assigned()) { + return nullptr; + } + return getInst(off); + } + + // Get a pointer to the instruction at offset |off| which must be within the + // bounds of the buffer. Use |getInstOrNull()| if |off| may be unassigned. + Inst* getInst(BufferOffset off) { + const int offset = off.getOffset(); + // This function is hot, do not make the next line a RELEASE_ASSERT. + MOZ_ASSERT(off.assigned() && offset >= 0 && unsigned(offset) < size()); + + // Is the instruction in the last slice? + if (offset >= int(bufferSize)) { + return (Inst*)&tail->instructions[offset - bufferSize]; + } + + // How close is this offset to the previous one we looked up? + // If it is sufficiently far from the start and end of the buffer, + // use the finger to start midway through the list. + int finger_dist = abs(offset - finger_offset); + if (finger_dist < std::min(offset, int(bufferSize - offset))) { + if (finger_offset < offset) { + return getInstForwards(off, finger, finger_offset, true); + } + return getInstBackwards(off, finger, finger_offset, true); + } + + // Is the instruction closer to the start or to the end? + if (offset < int(bufferSize - offset)) { + return getInstForwards(off, head, 0); + } + + // The last slice was already checked above, so start at the + // second-to-last. + Slice* prev = tail->getPrev(); + return getInstBackwards(off, prev, bufferSize - prev->length()); + } + + typedef AssemblerBuffer<SliceSize, Inst> ThisClass; + + class AssemblerBufferInstIterator { + BufferOffset bo_; + ThisClass* buffer_; + + public: + explicit AssemblerBufferInstIterator(BufferOffset bo, ThisClass* buffer) + : bo_(bo), buffer_(buffer) {} + void advance(int offset) { bo_ = BufferOffset(bo_.getOffset() + offset); } + Inst* next() { + advance(cur()->size()); + return cur(); + } + Inst* peek() { + return buffer_->getInst(BufferOffset(bo_.getOffset() + cur()->size())); + } + Inst* cur() const { return buffer_->getInst(bo_); } + }; +}; + +} // namespace jit +} // namespace js + +#endif // jit_shared_IonAssemblerBuffer_h diff --git a/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h b/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h new file mode 100644 index 0000000000..4f615db12f --- /dev/null +++ b/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h @@ -0,0 +1,1197 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_IonAssemblerBufferWithConstantPools_h +#define jit_shared_IonAssemblerBufferWithConstantPools_h + +#include "mozilla/CheckedInt.h" +#include "mozilla/MathAlgorithms.h" + +#include <algorithm> + +#include "jit/JitSpewer.h" +#include "jit/shared/IonAssemblerBuffer.h" + +// [SMDOC] JIT AssemblerBuffer constant pooling (ARM/ARM64/MIPS) +// +// This code extends the AssemblerBuffer to support the pooling of values loaded +// using program-counter relative addressing modes. This is necessary with the +// ARM instruction set because it has a fixed instruction size that can not +// encode all values as immediate arguments in instructions. Pooling the values +// allows the values to be placed in large chunks which minimizes the number of +// forced branches around them in the code. This is used for loading floating +// point constants, for loading 32 bit constants on the ARMv6, for absolute +// branch targets, and in future will be needed for large branches on the ARMv6. +// +// For simplicity of the implementation, the constant pools are always placed +// after the loads referencing them. When a new constant pool load is added to +// the assembler buffer, a corresponding pool entry is added to the current +// pending pool. The finishPool() method copies the current pending pool entries +// into the assembler buffer at the current offset and patches the pending +// constant pool load instructions. +// +// Before inserting instructions or pool entries, it is necessary to determine +// if doing so would place a pending pool entry out of reach of an instruction, +// and if so then the pool must firstly be dumped. With the allocation algorithm +// used below, the recalculation of all the distances between instructions and +// their pool entries can be avoided by noting that there will be a limiting +// instruction and pool entry pair that does not change when inserting more +// instructions. Adding more instructions makes the same increase to the +// distance, between instructions and their pool entries, for all such +// pairs. This pair is recorded as the limiter, and it is updated when new pool +// entries are added, see updateLimiter() +// +// The pools consist of: a guard instruction that branches around the pool, a +// header word that helps identify a pool in the instruction stream, and then +// the pool entries allocated in units of words. The guard instruction could be +// omitted if control does not reach the pool, and this is referred to as a +// natural guard below, but for simplicity the guard branch is always +// emitted. The pool header is an identifiable word that in combination with the +// guard uniquely identifies a pool in the instruction stream. The header also +// encodes the pool size and a flag indicating if the guard is natural. It is +// possible to iterate through the code instructions skipping or examining the +// pools. E.g. it might be necessary to skip pools when search for, or patching, +// an instruction sequence. +// +// It is often required to keep a reference to a pool entry, to patch it after +// the buffer is finished. Each pool entry is assigned a unique index, counting +// up from zero (see the poolEntryCount slot below). These can be mapped back to +// the offset of the pool entry in the finished buffer, see poolEntryOffset(). +// +// The code supports no-pool regions, and for these the size of the region, in +// instructions, must be supplied. This size is used to determine if inserting +// the instructions would place a pool entry out of range, and if so then a pool +// is firstly flushed. The DEBUG code checks that the emitted code is within the +// supplied size to detect programming errors. See enterNoPool() and +// leaveNoPool(). + +// The only planned instruction sets that require inline constant pools are the +// ARM, ARM64, and MIPS, and these all have fixed 32-bit sized instructions so +// for simplicity the code below is specialized for fixed 32-bit sized +// instructions and makes no attempt to support variable length +// instructions. The base assembler buffer which supports variable width +// instruction is used by the x86 and x64 backends. + +// The AssemblerBufferWithConstantPools template class uses static callbacks to +// the provided Asm template argument class: +// +// void Asm::InsertIndexIntoTag(uint8_t* load_, uint32_t index) +// +// When allocEntry() is called to add a constant pool load with an associated +// constant pool entry, this callback is called to encode the index of the +// allocated constant pool entry into the load instruction. +// +// After the constant pool has been placed, PatchConstantPoolLoad() is called +// to update the load instruction with the right load offset. +// +// void Asm::WritePoolGuard(BufferOffset branch, +// Instruction* dest, +// BufferOffset afterPool) +// +// Write out the constant pool guard branch before emitting the pool. +// +// branch +// Offset of the guard branch in the buffer. +// +// dest +// Pointer into the buffer where the guard branch should be emitted. (Same +// as getInst(branch)). Space for guardSize_ instructions has been reserved. +// +// afterPool +// Offset of the first instruction after the constant pool. This includes +// both pool entries and branch veneers added after the pool data. +// +// void Asm::WritePoolHeader(uint8_t* start, Pool* p, bool isNatural) +// +// Write out the pool header which follows the guard branch. +// +// void Asm::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr) +// +// Re-encode a load of a constant pool entry after the location of the +// constant pool is known. +// +// The load instruction at loadAddr was previously passed to +// InsertIndexIntoTag(). The constPoolAddr is the final address of the +// constant pool in the assembler buffer. +// +// void Asm::PatchShortRangeBranchToVeneer(AssemblerBufferWithConstantPools*, +// unsigned rangeIdx, +// BufferOffset deadline, +// BufferOffset veneer) +// +// Patch a short-range branch to jump through a veneer before it goes out of +// range. +// +// rangeIdx, deadline +// These arguments were previously passed to registerBranchDeadline(). It is +// assumed that PatchShortRangeBranchToVeneer() knows how to compute the +// offset of the short-range branch from this information. +// +// veneer +// Space for a branch veneer, guaranteed to be <= deadline. At this +// position, guardSize_ * InstSize bytes are allocated. They should be +// initialized to the proper unconditional branch instruction. +// +// Unbound branches to the same unbound label are organized as a linked list: +// +// Label::offset -> Branch1 -> Branch2 -> Branch3 -> nil +// +// This callback should insert a new veneer branch into the list: +// +// Label::offset -> Branch1 -> Branch2 -> Veneer -> Branch3 -> nil +// +// When Assembler::bind() rewrites the branches with the real label offset, it +// probably has to bind Branch2 to target the veneer branch instead of jumping +// straight to the label. + +namespace js { +namespace jit { + +// BranchDeadlineSet - Keep track of pending branch deadlines. +// +// Some architectures like arm and arm64 have branch instructions with limited +// range. When assembling a forward branch, it is not always known if the final +// target label will be in range of the branch instruction. +// +// The BranchDeadlineSet data structure is used to keep track of the set of +// pending forward branches. It supports the following fast operations: +// +// 1. Get the earliest deadline in the set. +// 2. Add a new branch deadline. +// 3. Remove a branch deadline. +// +// Architectures may have different branch encodings with different ranges. Each +// supported range is assigned a small integer starting at 0. This data +// structure does not care about the actual range of branch instructions, just +// the latest buffer offset that can be reached - the deadline offset. +// +// Branched are stored as (rangeIdx, deadline) tuples. The target-specific code +// can compute the location of the branch itself from this information. This +// data structure does not need to know. +// +template <unsigned NumRanges> +class BranchDeadlineSet { + // Maintain a list of pending deadlines for each range separately. + // + // The offsets in each vector are always kept in ascending order. + // + // Because we have a separate vector for different ranges, as forward + // branches are added to the assembler buffer, their deadlines will + // always be appended to the vector corresponding to their range. + // + // When binding labels, we expect a more-or-less LIFO order of branch + // resolutions. This would always hold if we had strictly structured control + // flow. + // + // We allow branch deadlines to be added and removed in any order, but + // performance is best in the expected case of near LIFO order. + // + typedef Vector<BufferOffset, 8, LifoAllocPolicy<Fallible>> RangeVector; + + // We really just want "RangeVector deadline_[NumRanges];", but each vector + // needs to be initialized with a LifoAlloc, and C++ doesn't bend that way. + // + // Use raw aligned storage instead and explicitly construct NumRanges + // vectors in our constructor. + mozilla::AlignedStorage2<RangeVector[NumRanges]> deadlineStorage_; + + // Always access the range vectors through this method. + RangeVector& vectorForRange(unsigned rangeIdx) { + MOZ_ASSERT(rangeIdx < NumRanges, "Invalid branch range index"); + return (*deadlineStorage_.addr())[rangeIdx]; + } + + const RangeVector& vectorForRange(unsigned rangeIdx) const { + MOZ_ASSERT(rangeIdx < NumRanges, "Invalid branch range index"); + return (*deadlineStorage_.addr())[rangeIdx]; + } + + // Maintain a precomputed earliest deadline at all times. + // This is unassigned only when all deadline vectors are empty. + BufferOffset earliest_; + + // The range vector owning earliest_. Uninitialized when empty. + unsigned earliestRange_; + + // Recompute the earliest deadline after it's been invalidated. + void recomputeEarliest() { + earliest_ = BufferOffset(); + for (unsigned r = 0; r < NumRanges; r++) { + auto& vec = vectorForRange(r); + if (!vec.empty() && (!earliest_.assigned() || vec[0] < earliest_)) { + earliest_ = vec[0]; + earliestRange_ = r; + } + } + } + + // Update the earliest deadline if needed after inserting (rangeIdx, + // deadline). Always return true for convenience: + // return insert() && updateEarliest(). + bool updateEarliest(unsigned rangeIdx, BufferOffset deadline) { + if (!earliest_.assigned() || deadline < earliest_) { + earliest_ = deadline; + earliestRange_ = rangeIdx; + } + return true; + } + + public: + explicit BranchDeadlineSet(LifoAlloc& alloc) : earliestRange_(0) { + // Manually construct vectors in the uninitialized aligned storage. + // This is because C++ arrays can otherwise only be constructed with + // the default constructor. + for (unsigned r = 0; r < NumRanges; r++) { + new (&vectorForRange(r)) RangeVector(alloc); + } + } + + ~BranchDeadlineSet() { + // Aligned storage doesn't destruct its contents automatically. + for (unsigned r = 0; r < NumRanges; r++) { + vectorForRange(r).~RangeVector(); + } + } + + // Is this set completely empty? + bool empty() const { return !earliest_.assigned(); } + + // Get the total number of deadlines in the set. + size_t size() const { + size_t count = 0; + for (unsigned r = 0; r < NumRanges; r++) { + count += vectorForRange(r).length(); + } + return count; + } + + // Get the number of deadlines for the range with the most elements. + size_t maxRangeSize() const { + size_t count = 0; + for (unsigned r = 0; r < NumRanges; r++) { + count = std::max(count, vectorForRange(r).length()); + } + return count; + } + + // Get the first deadline that is still in the set. + BufferOffset earliestDeadline() const { + MOZ_ASSERT(!empty()); + return earliest_; + } + + // Get the range index corresponding to earliestDeadlineRange(). + unsigned earliestDeadlineRange() const { + MOZ_ASSERT(!empty()); + return earliestRange_; + } + + // Add a (rangeIdx, deadline) tuple to the set. + // + // It is assumed that this tuple is not already in the set. + // This function performs best id the added deadline is later than any + // existing deadline for the same range index. + // + // Return true if the tuple was added, false if the tuple could not be added + // because of an OOM error. + bool addDeadline(unsigned rangeIdx, BufferOffset deadline) { + MOZ_ASSERT(deadline.assigned(), "Can only store assigned buffer offsets"); + // This is the vector where deadline should be saved. + auto& vec = vectorForRange(rangeIdx); + + // Fast case: Simple append to the relevant array. This never affects + // the earliest deadline. + if (!vec.empty() && vec.back() < deadline) { + return vec.append(deadline); + } + + // Fast case: First entry to the vector. We need to update earliest_. + if (vec.empty()) { + return vec.append(deadline) && updateEarliest(rangeIdx, deadline); + } + + return addDeadlineSlow(rangeIdx, deadline); + } + + private: + // General case of addDeadline. This is split into two functions such that + // the common case in addDeadline can be inlined while this part probably + // won't inline. + bool addDeadlineSlow(unsigned rangeIdx, BufferOffset deadline) { + auto& vec = vectorForRange(rangeIdx); + + // Inserting into the middle of the vector. Use a log time binary search + // and a linear time insert(). + // Is it worthwhile special-casing the empty vector? + auto at = std::lower_bound(vec.begin(), vec.end(), deadline); + MOZ_ASSERT(at == vec.end() || *at != deadline, + "Cannot insert duplicate deadlines"); + return vec.insert(at, deadline) && updateEarliest(rangeIdx, deadline); + } + + public: + // Remove a deadline from the set. + // If (rangeIdx, deadline) is not in the set, nothing happens. + void removeDeadline(unsigned rangeIdx, BufferOffset deadline) { + auto& vec = vectorForRange(rangeIdx); + + if (vec.empty()) { + return; + } + + if (deadline == vec.back()) { + // Expected fast case: Structured control flow causes forward + // branches to be bound in reverse order. + vec.popBack(); + } else { + // Slow case: Binary search + linear erase. + auto where = std::lower_bound(vec.begin(), vec.end(), deadline); + if (where == vec.end() || *where != deadline) { + return; + } + vec.erase(where); + } + if (deadline == earliest_) { + recomputeEarliest(); + } + } +}; + +// Specialization for architectures that don't need to track short-range +// branches. +template <> +class BranchDeadlineSet<0u> { + public: + explicit BranchDeadlineSet(LifoAlloc& alloc) {} + bool empty() const { return true; } + size_t size() const { return 0; } + size_t maxRangeSize() const { return 0; } + BufferOffset earliestDeadline() const { MOZ_CRASH(); } + unsigned earliestDeadlineRange() const { MOZ_CRASH(); } + bool addDeadline(unsigned rangeIdx, BufferOffset deadline) { MOZ_CRASH(); } + void removeDeadline(unsigned rangeIdx, BufferOffset deadline) { MOZ_CRASH(); } +}; + +// The allocation unit size for pools. +typedef int32_t PoolAllocUnit; + +// Hysteresis given to short-range branches. +// +// If any short-range branches will go out of range in the next N bytes, +// generate a veneer for them in the current pool. The hysteresis prevents the +// creation of many tiny constant pools for branch veneers. +const size_t ShortRangeBranchHysteresis = 128; + +struct Pool { + private: + // The maximum program-counter relative offset below which the instruction + // set can encode. Different classes of intructions might support different + // ranges but for simplicity the minimum is used here, and for the ARM this + // is constrained to 1024 by the float load instructions. + const size_t maxOffset_; + // An offset to apply to program-counter relative offsets. The ARM has a + // bias of 8. + const unsigned bias_; + + // The content of the pool entries. + Vector<PoolAllocUnit, 8, LifoAllocPolicy<Fallible>> poolData_; + + // Flag that tracks OOM conditions. This is set after any append failed. + bool oom_; + + // The limiting instruction and pool-entry pair. The instruction program + // counter relative offset of this limiting instruction will go out of range + // first as the pool position moves forward. It is more efficient to track + // just this limiting pair than to recheck all offsets when testing if the + // pool needs to be dumped. + // + // 1. The actual offset of the limiting instruction referencing the limiting + // pool entry. + BufferOffset limitingUser; + // 2. The pool entry index of the limiting pool entry. + unsigned limitingUsee; + + public: + // A record of the code offset of instructions that reference pool + // entries. These instructions need to be patched when the actual position + // of the instructions and pools are known, and for the code below this + // occurs when each pool is finished, see finishPool(). + Vector<BufferOffset, 8, LifoAllocPolicy<Fallible>> loadOffsets; + + // Create a Pool. Don't allocate anything from lifoAloc, just capture its + // reference. + explicit Pool(size_t maxOffset, unsigned bias, LifoAlloc& lifoAlloc) + : maxOffset_(maxOffset), + bias_(bias), + poolData_(lifoAlloc), + oom_(false), + limitingUser(), + limitingUsee(INT_MIN), + loadOffsets(lifoAlloc) {} + + // If poolData() returns nullptr then oom_ will also be true. + const PoolAllocUnit* poolData() const { return poolData_.begin(); } + + unsigned numEntries() const { return poolData_.length(); } + + size_t getPoolSize() const { return numEntries() * sizeof(PoolAllocUnit); } + + bool oom() const { return oom_; } + + // Update the instruction/pool-entry pair that limits the position of the + // pool. The nextInst is the actual offset of the new instruction being + // allocated. + // + // This is comparing the offsets, see checkFull() below for the equation, + // but common expressions on both sides have been canceled from the ranges + // being compared. Notably, the poolOffset cancels out, so the limiting pair + // does not depend on where the pool is placed. + void updateLimiter(BufferOffset nextInst) { + ptrdiff_t oldRange = + limitingUsee * sizeof(PoolAllocUnit) - limitingUser.getOffset(); + ptrdiff_t newRange = getPoolSize() - nextInst.getOffset(); + if (!limitingUser.assigned() || newRange > oldRange) { + // We have a new largest range! + limitingUser = nextInst; + limitingUsee = numEntries(); + } + } + + // Check if inserting a pool at the actual offset poolOffset would place + // pool entries out of reach. This is called before inserting instructions + // to check that doing so would not push pool entries out of reach, and if + // so then the pool would need to be firstly dumped. The poolOffset is the + // first word of the pool, after the guard and header and alignment fill. + bool checkFull(size_t poolOffset) const { + // Not full if there are no uses. + if (!limitingUser.assigned()) { + return false; + } + size_t offset = poolOffset + limitingUsee * sizeof(PoolAllocUnit) - + (limitingUser.getOffset() + bias_); + return offset >= maxOffset_; + } + + static const unsigned OOM_FAIL = unsigned(-1); + + unsigned insertEntry(unsigned num, uint8_t* data, BufferOffset off, + LifoAlloc& lifoAlloc) { + if (oom_) { + return OOM_FAIL; + } + unsigned ret = numEntries(); + if (!poolData_.append((PoolAllocUnit*)data, num) || + !loadOffsets.append(off)) { + oom_ = true; + return OOM_FAIL; + } + return ret; + } + + void reset() { + poolData_.clear(); + loadOffsets.clear(); + + limitingUser = BufferOffset(); + limitingUsee = -1; + } +}; + +// Template arguments: +// +// SliceSize +// Number of bytes in each allocated BufferSlice. See +// AssemblerBuffer::SliceSize. +// +// InstSize +// Size in bytes of the fixed-size instructions. This should be equal to +// sizeof(Inst). This is only needed here because the buffer is defined before +// the Instruction. +// +// Inst +// The actual type used to represent instructions. This is only really used as +// the return type of the getInst() method. +// +// Asm +// Class defining the needed static callback functions. See documentation of +// the Asm::* callbacks above. +// +// NumShortBranchRanges +// The number of short branch ranges to support. This can be 0 if no support +// for tracking short range branches is needed. The +// AssemblerBufferWithConstantPools class does not need to know what the range +// of branches is - it deals in branch 'deadlines' which is the last buffer +// position that a short-range forward branch can reach. It is assumed that +// the Asm class is able to find the actual branch instruction given a +// (range-index, deadline) pair. +// +// +template <size_t SliceSize, size_t InstSize, class Inst, class Asm, + unsigned NumShortBranchRanges = 0> +struct AssemblerBufferWithConstantPools + : public AssemblerBuffer<SliceSize, Inst> { + private: + // The PoolEntry index counter. Each PoolEntry is given a unique index, + // counting up from zero, and these can be mapped back to the actual pool + // entry offset after finishing the buffer, see poolEntryOffset(). + size_t poolEntryCount; + + public: + class PoolEntry { + size_t index_; + + public: + explicit PoolEntry(size_t index) : index_(index) {} + + PoolEntry() : index_(-1) {} + + size_t index() const { return index_; } + }; + + private: + typedef AssemblerBuffer<SliceSize, Inst> Parent; + using typename Parent::Slice; + + // The size of a pool guard, in instructions. A branch around the pool. + const unsigned guardSize_; + // The size of the header that is put at the beginning of a full pool, in + // instruction sized units. + const unsigned headerSize_; + + // The maximum pc relative offset encoded in instructions that reference + // pool entries. This is generally set to the maximum offset that can be + // encoded by the instructions, but for testing can be lowered to affect the + // pool placement and frequency of pool placement. + const size_t poolMaxOffset_; + + // The bias on pc relative addressing mode offsets, in units of bytes. The + // ARM has a bias of 8 bytes. + const unsigned pcBias_; + + // The current working pool. Copied out as needed before resetting. + Pool pool_; + + // The buffer should be aligned to this address. + const size_t instBufferAlign_; + + struct PoolInfo { + // The index of the first entry in this pool. + // Pool entries are numbered uniquely across all pools, starting from 0. + unsigned firstEntryIndex; + + // The location of this pool's first entry in the main assembler buffer. + // Note that the pool guard and header come before this offset which + // points directly at the data. + BufferOffset offset; + + explicit PoolInfo(unsigned index, BufferOffset data) + : firstEntryIndex(index), offset(data) {} + }; + + // Info for each pool that has already been dumped. This does not include + // any entries in pool_. + Vector<PoolInfo, 8, LifoAllocPolicy<Fallible>> poolInfo_; + + // Set of short-range forward branches that have not yet been bound. + // We may need to insert veneers if the final label turns out to be out of + // range. + // + // This set stores (rangeIdx, deadline) pairs instead of the actual branch + // locations. + BranchDeadlineSet<NumShortBranchRanges> branchDeadlines_; + + // When true dumping pools is inhibited. + bool canNotPlacePool_; + +#ifdef DEBUG + // State for validating the 'maxInst' argument to enterNoPool(). + // The buffer offset when entering the no-pool region. + size_t canNotPlacePoolStartOffset_; + // The maximum number of word sized instructions declared for the no-pool + // region. + size_t canNotPlacePoolMaxInst_; +#endif + + // Instruction to use for alignment fill. + const uint32_t alignFillInst_; + + // Insert a number of NOP instructions between each requested instruction at + // all locations at which a pool can potentially spill. This is useful for + // checking that instruction locations are correctly referenced and/or + // followed. + const uint32_t nopFillInst_; + const unsigned nopFill_; + + // For inhibiting the insertion of fill NOPs in the dynamic context in which + // they are being inserted. + bool inhibitNops_; + + private: + // The buffer slices are in a double linked list. + Slice* getHead() const { return this->head; } + Slice* getTail() const { return this->tail; } + + public: + AssemblerBufferWithConstantPools(unsigned guardSize, unsigned headerSize, + size_t instBufferAlign, size_t poolMaxOffset, + unsigned pcBias, uint32_t alignFillInst, + uint32_t nopFillInst, unsigned nopFill = 0) + : poolEntryCount(0), + guardSize_(guardSize), + headerSize_(headerSize), + poolMaxOffset_(poolMaxOffset), + pcBias_(pcBias), + pool_(poolMaxOffset, pcBias, this->lifoAlloc_), + instBufferAlign_(instBufferAlign), + poolInfo_(this->lifoAlloc_), + branchDeadlines_(this->lifoAlloc_), + canNotPlacePool_(false), +#ifdef DEBUG + canNotPlacePoolStartOffset_(0), + canNotPlacePoolMaxInst_(0), +#endif + alignFillInst_(alignFillInst), + nopFillInst_(nopFillInst), + nopFill_(nopFill), + inhibitNops_(false) { + } + + private: + size_t sizeExcludingCurrentPool() const { + // Return the actual size of the buffer, excluding the current pending + // pool. + return this->nextOffset().getOffset(); + } + + public: + size_t size() const { + // Return the current actual size of the buffer. This is only accurate + // if there are no pending pool entries to dump, check. + MOZ_ASSERT_IF(!this->oom(), pool_.numEntries() == 0); + return sizeExcludingCurrentPool(); + } + + private: + void insertNopFill() { + // Insert fill for testing. + if (nopFill_ > 0 && !inhibitNops_ && !canNotPlacePool_) { + inhibitNops_ = true; + + // Fill using a branch-nop rather than a NOP so this can be + // distinguished and skipped. + for (size_t i = 0; i < nopFill_; i++) { + putInt(nopFillInst_); + } + + inhibitNops_ = false; + } + } + + static const unsigned OOM_FAIL = unsigned(-1); + static const unsigned DUMMY_INDEX = unsigned(-2); + + // Check if it is possible to add numInst instructions and numPoolEntries + // constant pool entries without needing to flush the current pool. + bool hasSpaceForInsts(unsigned numInsts, unsigned numPoolEntries) const { + size_t nextOffset = sizeExcludingCurrentPool(); + // Earliest starting offset for the current pool after adding numInsts. + // This is the beginning of the pool entries proper, after inserting a + // guard branch + pool header. + size_t poolOffset = + nextOffset + (numInsts + guardSize_ + headerSize_) * InstSize; + + // Any constant pool loads that would go out of range? + if (pool_.checkFull(poolOffset)) { + return false; + } + + // Any short-range branch that would go out of range? + if (!branchDeadlines_.empty()) { + size_t deadline = branchDeadlines_.earliestDeadline().getOffset(); + size_t poolEnd = poolOffset + pool_.getPoolSize() + + numPoolEntries * sizeof(PoolAllocUnit); + + // When NumShortBranchRanges > 1, is is possible for branch deadlines to + // expire faster than we can insert veneers. Suppose branches are 4 bytes + // each, we could have the following deadline set: + // + // Range 0: 40, 44, 48 + // Range 1: 44, 48 + // + // It is not good enough to start inserting veneers at the 40 deadline; we + // would not be able to create veneers for the second 44 deadline. + // Instead, we need to start at 32: + // + // 32: veneer(40) + // 36: veneer(44) + // 40: veneer(44) + // 44: veneer(48) + // 48: veneer(48) + // + // This is a pretty conservative solution to the problem: If we begin at + // the earliest deadline, we can always emit all veneers for the range + // that currently has the most pending deadlines. That may not leave room + // for veneers for the remaining ranges, so reserve space for those + // secondary range veneers assuming the worst case deadlines. + + // Total pending secondary range veneer size. + size_t secondaryVeneers = guardSize_ * (branchDeadlines_.size() - + branchDeadlines_.maxRangeSize()); + + if (deadline < poolEnd + secondaryVeneers) { + return false; + } + } + + return true; + } + + unsigned insertEntryForwards(unsigned numInst, unsigned numPoolEntries, + uint8_t* inst, uint8_t* data) { + // If inserting pool entries then find a new limiter before we do the + // range check. + if (numPoolEntries) { + pool_.updateLimiter(BufferOffset(sizeExcludingCurrentPool())); + } + + if (!hasSpaceForInsts(numInst, numPoolEntries)) { + if (numPoolEntries) { + JitSpew(JitSpew_Pools, "Inserting pool entry caused a spill"); + } else { + JitSpew(JitSpew_Pools, "Inserting instruction(%zu) caused a spill", + sizeExcludingCurrentPool()); + } + + finishPool(numInst * InstSize); + if (this->oom()) { + return OOM_FAIL; + } + return insertEntryForwards(numInst, numPoolEntries, inst, data); + } + if (numPoolEntries) { + unsigned result = pool_.insertEntry(numPoolEntries, data, + this->nextOffset(), this->lifoAlloc_); + if (result == Pool::OOM_FAIL) { + this->fail_oom(); + return OOM_FAIL; + } + return result; + } + + // The pool entry index is returned above when allocating an entry, but + // when not allocating an entry a dummy value is returned - it is not + // expected to be used by the caller. + return DUMMY_INDEX; + } + + public: + // Get the next buffer offset where an instruction would be inserted. + // This may flush the current constant pool before returning nextOffset(). + BufferOffset nextInstrOffset(int numInsts = 1) { + if (!hasSpaceForInsts(numInsts, /* numPoolEntries= */ 0)) { + JitSpew(JitSpew_Pools, + "nextInstrOffset @ %d caused a constant pool spill", + this->nextOffset().getOffset()); + finishPool(ShortRangeBranchHysteresis); + } + return this->nextOffset(); + } + + MOZ_NEVER_INLINE + BufferOffset allocEntry(size_t numInst, unsigned numPoolEntries, + uint8_t* inst, uint8_t* data, + PoolEntry* pe = nullptr) { + // The allocation of pool entries is not supported in a no-pool region, + // check. + MOZ_ASSERT_IF(numPoolEntries, !canNotPlacePool_); + + if (this->oom()) { + return BufferOffset(); + } + + insertNopFill(); + +#ifdef JS_JITSPEW + if (numPoolEntries && JitSpewEnabled(JitSpew_Pools)) { + JitSpew(JitSpew_Pools, "Inserting %d entries into pool", numPoolEntries); + JitSpewStart(JitSpew_Pools, "data is: 0x"); + size_t length = numPoolEntries * sizeof(PoolAllocUnit); + for (unsigned idx = 0; idx < length; idx++) { + JitSpewCont(JitSpew_Pools, "%02x", data[length - idx - 1]); + if (((idx & 3) == 3) && (idx + 1 != length)) { + JitSpewCont(JitSpew_Pools, "_"); + } + } + JitSpewFin(JitSpew_Pools); + } +#endif + + // Insert the pool value. + unsigned index = insertEntryForwards(numInst, numPoolEntries, inst, data); + if (this->oom()) { + return BufferOffset(); + } + + // Now to get an instruction to write. + PoolEntry retPE; + if (numPoolEntries) { + JitSpew(JitSpew_Pools, "Entry has index %u, offset %zu", index, + sizeExcludingCurrentPool()); + Asm::InsertIndexIntoTag(inst, index); + // Figure out the offset within the pool entries. + retPE = PoolEntry(poolEntryCount); + poolEntryCount += numPoolEntries; + } + // Now inst is a valid thing to insert into the instruction stream. + if (pe != nullptr) { + *pe = retPE; + } + return this->putBytes(numInst * InstSize, inst); + } + + // putInt is the workhorse for the assembler and higher-level buffer + // abstractions: it places one instruction into the instruction stream. + // Under normal circumstances putInt should just check that the constant + // pool does not need to be flushed, that there is space for the single word + // of the instruction, and write that word and update the buffer pointer. + // + // To do better here we need a status variable that handles both nopFill_ + // and capacity, so that we can quickly know whether to go the slow path. + // That could be a variable that has the remaining number of simple + // instructions that can be inserted before a more expensive check, + // which is set to zero when nopFill_ is set. + // + // We assume that we don't have to check this->oom() if there is space to + // insert a plain instruction; there will always come a later time when it + // will be checked anyway. + + MOZ_ALWAYS_INLINE + BufferOffset putInt(uint32_t value) { + if (nopFill_ || + !hasSpaceForInsts(/* numInsts= */ 1, /* numPoolEntries= */ 0)) { + return allocEntry(1, 0, (uint8_t*)&value, nullptr, nullptr); + } + +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) + return this->putU32Aligned(value); +#else + return this->AssemblerBuffer<SliceSize, Inst>::putInt(value); +#endif + } + + // Register a short-range branch deadline. + // + // After inserting a short-range forward branch, call this method to + // register the branch 'deadline' which is the last buffer offset that the + // branch instruction can reach. + // + // When the branch is bound to a destination label, call + // unregisterBranchDeadline() to stop tracking this branch, + // + // If the assembled code is about to exceed the registered branch deadline, + // and unregisterBranchDeadline() has not yet been called, an + // instruction-sized constant pool entry is allocated before the branch + // deadline. + // + // rangeIdx + // A number < NumShortBranchRanges identifying the range of the branch. + // + // deadline + // The highest buffer offset the the short-range branch can reach + // directly. + // + void registerBranchDeadline(unsigned rangeIdx, BufferOffset deadline) { + if (!this->oom() && !branchDeadlines_.addDeadline(rangeIdx, deadline)) { + this->fail_oom(); + } + } + + // Un-register a short-range branch deadline. + // + // When a short-range branch has been successfully bound to its destination + // label, call this function to stop traching the branch. + // + // The (rangeIdx, deadline) pair must be previously registered. + // + void unregisterBranchDeadline(unsigned rangeIdx, BufferOffset deadline) { + if (!this->oom()) { + branchDeadlines_.removeDeadline(rangeIdx, deadline); + } + } + + private: + // Are any short-range branches about to expire? + bool hasExpirableShortRangeBranches(size_t reservedBytes) const { + if (branchDeadlines_.empty()) { + return false; + } + + // Include branches that would expire in the next N bytes. The reservedBytes + // argument avoids the needless creation of many tiny constant pools. + // + // As the reservedBytes could be of any sizes such as SIZE_MAX, in the case + // of flushPool, we have to check for overflow when comparing the deadline + // with our expected reserved bytes. + size_t deadline = branchDeadlines_.earliestDeadline().getOffset(); + using CheckedSize = mozilla::CheckedInt<size_t>; + CheckedSize current(this->nextOffset().getOffset()); + CheckedSize poolFreeSpace(reservedBytes); + auto future = current + poolFreeSpace; + return !future.isValid() || deadline < future.value(); + } + + bool isPoolEmptyFor(size_t bytes) const { + return pool_.numEntries() == 0 && !hasExpirableShortRangeBranches(bytes); + } + void finishPool(size_t reservedBytes) { + JitSpew(JitSpew_Pools, "Attempting to finish pool %zu with %u entries.", + poolInfo_.length(), pool_.numEntries()); + + if (reservedBytes < ShortRangeBranchHysteresis) { + reservedBytes = ShortRangeBranchHysteresis; + } + + if (isPoolEmptyFor(reservedBytes)) { + // If there is no data in the pool being dumped, don't dump anything. + JitSpew(JitSpew_Pools, "Aborting because the pool is empty"); + return; + } + + // Should not be placing a pool in a no-pool region, check. + MOZ_ASSERT(!canNotPlacePool_); + + // Dump the pool with a guard branch around the pool. + BufferOffset guard = this->putBytes(guardSize_ * InstSize, nullptr); + BufferOffset header = this->putBytes(headerSize_ * InstSize, nullptr); + BufferOffset data = this->putBytesLarge(pool_.getPoolSize(), + (const uint8_t*)pool_.poolData()); + if (this->oom()) { + return; + } + + // Now generate branch veneers for any short-range branches that are + // about to expire. + while (hasExpirableShortRangeBranches(reservedBytes)) { + unsigned rangeIdx = branchDeadlines_.earliestDeadlineRange(); + BufferOffset deadline = branchDeadlines_.earliestDeadline(); + + // Stop tracking this branch. The Asm callback below may register + // new branches to track. + branchDeadlines_.removeDeadline(rangeIdx, deadline); + + // Make room for the veneer. Same as a pool guard branch. + BufferOffset veneer = this->putBytes(guardSize_ * InstSize, nullptr); + if (this->oom()) { + return; + } + + // Fix the branch so it targets the veneer. + // The Asm class knows how to find the original branch given the + // (rangeIdx, deadline) pair. + Asm::PatchShortRangeBranchToVeneer(this, rangeIdx, deadline, veneer); + } + + // We only reserved space for the guard branch and pool header. + // Fill them in. + BufferOffset afterPool = this->nextOffset(); + Asm::WritePoolGuard(guard, this->getInst(guard), afterPool); + Asm::WritePoolHeader((uint8_t*)this->getInst(header), &pool_, false); + + // With the pool's final position determined it is now possible to patch + // the instructions that reference entries in this pool, and this is + // done incrementally as each pool is finished. + size_t poolOffset = data.getOffset(); + + unsigned idx = 0; + for (BufferOffset* iter = pool_.loadOffsets.begin(); + iter != pool_.loadOffsets.end(); ++iter, ++idx) { + // All entries should be before the pool. + MOZ_ASSERT(iter->getOffset() < guard.getOffset()); + + // Everything here is known so we can safely do the necessary + // substitutions. + Inst* inst = this->getInst(*iter); + size_t codeOffset = poolOffset - iter->getOffset(); + + // That is, PatchConstantPoolLoad wants to be handed the address of + // the pool entry that is being loaded. We need to do a non-trivial + // amount of math here, since the pool that we've made does not + // actually reside there in memory. + JitSpew(JitSpew_Pools, "Fixing entry %d offset to %zu", idx, codeOffset); + Asm::PatchConstantPoolLoad(inst, (uint8_t*)inst + codeOffset); + } + + // Record the pool info. + unsigned firstEntry = poolEntryCount - pool_.numEntries(); + if (!poolInfo_.append(PoolInfo(firstEntry, data))) { + this->fail_oom(); + return; + } + + // Reset everything to the state that it was in when we started. + pool_.reset(); + } + + public: + void flushPool() { + if (this->oom()) { + return; + } + JitSpew(JitSpew_Pools, "Requesting a pool flush"); + finishPool(SIZE_MAX); + } + + void enterNoPool(size_t maxInst) { + if (this->oom()) { + return; + } + // Don't allow re-entry. + MOZ_ASSERT(!canNotPlacePool_); + insertNopFill(); + + // Check if the pool will spill by adding maxInst instructions, and if + // so then finish the pool before entering the no-pool region. It is + // assumed that no pool entries are allocated in a no-pool region and + // this is asserted when allocating entries. + if (!hasSpaceForInsts(maxInst, 0)) { + JitSpew(JitSpew_Pools, "No-Pool instruction(%zu) caused a spill.", + sizeExcludingCurrentPool()); + finishPool(maxInst * InstSize); + if (this->oom()) { + return; + } + MOZ_ASSERT(hasSpaceForInsts(maxInst, 0)); + } + +#ifdef DEBUG + // Record the buffer position to allow validating maxInst when leaving + // the region. + canNotPlacePoolStartOffset_ = this->nextOffset().getOffset(); + canNotPlacePoolMaxInst_ = maxInst; +#endif + + canNotPlacePool_ = true; + } + + void leaveNoPool() { + if (this->oom()) { + canNotPlacePool_ = false; + return; + } + MOZ_ASSERT(canNotPlacePool_); + canNotPlacePool_ = false; + + // Validate the maxInst argument supplied to enterNoPool(). + MOZ_ASSERT(this->nextOffset().getOffset() - canNotPlacePoolStartOffset_ <= + canNotPlacePoolMaxInst_ * InstSize); + } + + void enterNoNops() { + MOZ_ASSERT(!inhibitNops_); + inhibitNops_ = true; + } + void leaveNoNops() { + MOZ_ASSERT(inhibitNops_); + inhibitNops_ = false; + } + void assertNoPoolAndNoNops() { + MOZ_ASSERT(inhibitNops_); + MOZ_ASSERT_IF(!this->oom(), isPoolEmptyFor(InstSize) || canNotPlacePool_); + } + + void align(unsigned alignment) { align(alignment, alignFillInst_); } + + void align(unsigned alignment, uint32_t pattern) { + MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment)); + MOZ_ASSERT(alignment >= InstSize); + + // A pool many need to be dumped at this point, so insert NOP fill here. + insertNopFill(); + + // Check if the code position can be aligned without dumping a pool. + unsigned requiredFill = sizeExcludingCurrentPool() & (alignment - 1); + if (requiredFill == 0) { + return; + } + requiredFill = alignment - requiredFill; + + // Add an InstSize because it is probably not useful for a pool to be + // dumped at the aligned code position. + if (!hasSpaceForInsts(requiredFill / InstSize + 1, 0)) { + // Alignment would cause a pool dump, so dump the pool now. + JitSpew(JitSpew_Pools, "Alignment of %d at %zu caused a spill.", + alignment, sizeExcludingCurrentPool()); + finishPool(requiredFill); + } + + bool prevInhibitNops = inhibitNops_; + inhibitNops_ = true; + while ((sizeExcludingCurrentPool() & (alignment - 1)) && !this->oom()) { + putInt(pattern); + } + inhibitNops_ = prevInhibitNops; + } + + public: + void executableCopy(uint8_t* dest) { + if (this->oom()) { + return; + } + // The pools should have all been flushed, check. + MOZ_ASSERT(pool_.numEntries() == 0); + for (Slice* cur = getHead(); cur != nullptr; cur = cur->getNext()) { + memcpy(dest, &cur->instructions[0], cur->length()); + dest += cur->length(); + } + } + + bool appendRawCode(const uint8_t* code, size_t numBytes) { + if (this->oom()) { + return false; + } + // The pools should have all been flushed, check. + MOZ_ASSERT(pool_.numEntries() == 0); + while (numBytes > SliceSize) { + this->putBytes(SliceSize, code); + numBytes -= SliceSize; + code += SliceSize; + } + this->putBytes(numBytes, code); + return !this->oom(); + } + + public: + size_t poolEntryOffset(PoolEntry pe) const { + MOZ_ASSERT(pe.index() < poolEntryCount - pool_.numEntries(), + "Invalid pool entry, or not flushed yet."); + // Find the pool containing pe.index(). + // The array is sorted, so we can use a binary search. + auto b = poolInfo_.begin(), e = poolInfo_.end(); + // A note on asymmetric types in the upper_bound comparator: + // http://permalink.gmane.org/gmane.comp.compilers.clang.devel/10101 + auto i = std::upper_bound(b, e, pe.index(), + [](size_t value, const PoolInfo& entry) { + return value < entry.firstEntryIndex; + }); + // Since upper_bound finds the first pool greater than pe, + // we want the previous one which is the last one less than or equal. + MOZ_ASSERT(i != b, "PoolInfo not sorted or empty?"); + --i; + // The i iterator now points to the pool containing pe.index. + MOZ_ASSERT(i->firstEntryIndex <= pe.index() && + (i + 1 == e || (i + 1)->firstEntryIndex > pe.index())); + // Compute the byte offset into the pool. + unsigned relativeIndex = pe.index() - i->firstEntryIndex; + return i->offset.getOffset() + relativeIndex * sizeof(PoolAllocUnit); + } +}; + +} // namespace jit +} // namespace js + +#endif // jit_shared_IonAssemblerBufferWithConstantPools_h diff --git a/js/src/jit/shared/LIR-shared.h b/js/src/jit/shared/LIR-shared.h new file mode 100644 index 0000000000..03b0ccef53 --- /dev/null +++ b/js/src/jit/shared/LIR-shared.h @@ -0,0 +1,4272 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_LIR_shared_h +#define jit_shared_LIR_shared_h + +#include "mozilla/Maybe.h" +#include "jit/AtomicOp.h" +#include "jit/shared/Assembler-shared.h" +#include "util/Memory.h" + +// This file declares LIR instructions that are common to every platform. + +namespace js { +namespace jit { + +LIR_OPCODE_CLASS_GENERATED + +#ifdef FUZZING_JS_FUZZILLI +class LFuzzilliHashT : public LInstructionHelper<1, 1, 2> { + public: + LIR_HEADER(FuzzilliHashT); + + LFuzzilliHashT(const LAllocation& value, const LDefinition& temp, + const LDefinition& tempFloat) + : LInstructionHelper(classOpcode) { + setOperand(0, value); + setTemp(0, temp); + setTemp(1, tempFloat); + } + + const LAllocation* value() { return getOperand(0); } + + MFuzzilliHash* mir() const { return mir_->toFuzzilliHash(); } +}; + +class LFuzzilliHashV : public LInstructionHelper<1, BOX_PIECES, 2> { + public: + LIR_HEADER(FuzzilliHashV); + + LFuzzilliHashV(const LBoxAllocation& value, const LDefinition& temp, + const LDefinition& tempFloat) + : LInstructionHelper(classOpcode) { + setBoxOperand(0, value); + setTemp(0, temp); + setTemp(1, tempFloat); + } + + MFuzzilliHash* mir() const { return mir_->toFuzzilliHash(); } +}; + +class LFuzzilliHashStore : public LInstructionHelper<0, 1, 2> { + public: + LIR_HEADER(FuzzilliHashStore); + + LFuzzilliHashStore(const LAllocation& value, const LDefinition& temp1, + const LDefinition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, value); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LAllocation* value() { return getOperand(0); } + + MFuzzilliHashStore* mir() const { return mir_->toFuzzilliHashStore(); } +}; +#endif + +class LBox : public LInstructionHelper<BOX_PIECES, 1, 0> { + MIRType type_; + + public: + LIR_HEADER(Box); + + LBox(const LAllocation& payload, MIRType type) + : LInstructionHelper(classOpcode), type_(type) { + setOperand(0, payload); + } + + MIRType type() const { return type_; } + const char* extraName() const { return StringFromMIRType(type_); } +}; + +template <size_t Temps, size_t ExtraUses = 0> +class LBinaryMath : public LInstructionHelper<1, 2 + ExtraUses, Temps> { + protected: + explicit LBinaryMath(LNode::Opcode opcode) + : LInstructionHelper<1, 2 + ExtraUses, Temps>(opcode) {} + + public: + const LAllocation* lhs() { return this->getOperand(0); } + const LAllocation* rhs() { return this->getOperand(1); } +}; + +template <size_t Temps, size_t ExtraUses = 0> +class LUnaryMath : public LInstructionHelper<1, 1 + ExtraUses, Temps> { + protected: + explicit LUnaryMath(LNode::Opcode opcode) + : LInstructionHelper<1, 1 + ExtraUses, Temps>(opcode) {} + + public: + const LAllocation* input() { return this->getOperand(0); } +}; + +// An LOsiPoint captures a snapshot after a call and ensures enough space to +// patch in a call to the invalidation mechanism. +// +// Note: LSafepoints are 1:1 with LOsiPoints, so it holds a reference to the +// corresponding LSafepoint to inform it of the LOsiPoint's masm offset when it +// gets GC'd. +class LOsiPoint : public LInstructionHelper<0, 0, 0> { + LSafepoint* safepoint_; + + public: + LOsiPoint(LSafepoint* safepoint, LSnapshot* snapshot) + : LInstructionHelper(classOpcode), safepoint_(safepoint) { + MOZ_ASSERT(safepoint && snapshot); + assignSnapshot(snapshot); + } + + LSafepoint* associatedSafepoint() { return safepoint_; } + + LIR_HEADER(OsiPoint) +}; + +class LMove { + LAllocation from_; + LAllocation to_; + LDefinition::Type type_; + + public: + LMove(LAllocation from, LAllocation to, LDefinition::Type type) + : from_(from), to_(to), type_(type) {} + + LAllocation from() const { return from_; } + LAllocation to() const { return to_; } + LDefinition::Type type() const { return type_; } +}; + +class LMoveGroup : public LInstructionHelper<0, 0, 0> { + js::Vector<LMove, 2, JitAllocPolicy> moves_; + +#ifdef JS_CODEGEN_X86 + // Optional general register available for use when executing moves. + LAllocation scratchRegister_; +#endif + + explicit LMoveGroup(TempAllocator& alloc) + : LInstructionHelper(classOpcode), moves_(alloc) {} + + public: + LIR_HEADER(MoveGroup) + + static LMoveGroup* New(TempAllocator& alloc) { + return new (alloc) LMoveGroup(alloc); + } + + void printOperands(GenericPrinter& out); + + // Add a move which takes place simultaneously with all others in the group. + bool add(LAllocation from, LAllocation to, LDefinition::Type type); + + // Add a move which takes place after existing moves in the group. + bool addAfter(LAllocation from, LAllocation to, LDefinition::Type type); + + size_t numMoves() const { return moves_.length(); } + const LMove& getMove(size_t i) const { return moves_[i]; } + +#ifdef JS_CODEGEN_X86 + void setScratchRegister(Register reg) { scratchRegister_ = LGeneralReg(reg); } + LAllocation maybeScratchRegister() { return scratchRegister_; } +#endif + + bool uses(Register reg) { + for (size_t i = 0; i < numMoves(); i++) { + LMove move = getMove(i); + if (move.from() == LGeneralReg(reg) || move.to() == LGeneralReg(reg)) { + return true; + } + } + return false; + } +}; + +// A constant Value. +class LValue : public LInstructionHelper<BOX_PIECES, 0, 0> { + Value v_; + + public: + LIR_HEADER(Value) + + explicit LValue(const Value& v) : LInstructionHelper(classOpcode), v_(v) {} + + Value value() const { return v_; } +}; + +// Base class for control instructions (goto, branch, etc.) +template <size_t Succs, size_t Operands, size_t Temps> +class LControlInstructionHelper + : public LInstructionHelper<0, Operands, Temps> { + mozilla::Array<MBasicBlock*, Succs> successors_; + + protected: + explicit LControlInstructionHelper(LNode::Opcode opcode) + : LInstructionHelper<0, Operands, Temps>(opcode) {} + + public: + size_t numSuccessors() const { return Succs; } + MBasicBlock* getSuccessor(size_t i) const { return successors_[i]; } + + void setSuccessor(size_t i, MBasicBlock* successor) { + successors_[i] = successor; + } +}; + +// Jumps to the start of a basic block. +class LGoto : public LControlInstructionHelper<1, 0, 0> { + public: + LIR_HEADER(Goto) + + explicit LGoto(MBasicBlock* block) : LControlInstructionHelper(classOpcode) { + setSuccessor(0, block); + } + + MBasicBlock* target() const { return getSuccessor(0); } +}; + +class LNewArray : public LInstructionHelper<1, 0, 1> { + public: + LIR_HEADER(NewArray) + + explicit LNewArray(const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setTemp(0, temp); + } + + const char* extraName() const { + return mir()->isVMCall() ? "VMCall" : nullptr; + } + + const LDefinition* temp() { return getTemp(0); } + + MNewArray* mir() const { return mir_->toNewArray(); } +}; + +class LNewObject : public LInstructionHelper<1, 0, 1> { + public: + LIR_HEADER(NewObject) + + explicit LNewObject(const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setTemp(0, temp); + } + + const char* extraName() const { + return mir()->isVMCall() ? "VMCall" : nullptr; + } + + const LDefinition* temp() { return getTemp(0); } + + MNewObject* mir() const { return mir_->toNewObject(); } +}; + +template <size_t Defs, size_t Ops> +class LWasmReinterpretBase : public LInstructionHelper<Defs, Ops, 0> { + typedef LInstructionHelper<Defs, Ops, 0> Base; + + protected: + explicit LWasmReinterpretBase(LNode::Opcode opcode) : Base(opcode) {} + + public: + const LAllocation* input() { return Base::getOperand(0); } + MWasmReinterpret* mir() const { return Base::mir_->toWasmReinterpret(); } +}; + +class LWasmReinterpret : public LWasmReinterpretBase<1, 1> { + public: + LIR_HEADER(WasmReinterpret); + explicit LWasmReinterpret(const LAllocation& input) + : LWasmReinterpretBase(classOpcode) { + setOperand(0, input); + } +}; + +class LWasmReinterpretFromI64 : public LWasmReinterpretBase<1, INT64_PIECES> { + public: + static const size_t Input = 0; + + LIR_HEADER(WasmReinterpretFromI64); + explicit LWasmReinterpretFromI64(const LInt64Allocation& input) + : LWasmReinterpretBase(classOpcode) { + setInt64Operand(Input, input); + } +}; + +class LWasmReinterpretToI64 : public LWasmReinterpretBase<INT64_PIECES, 1> { + public: + LIR_HEADER(WasmReinterpretToI64); + explicit LWasmReinterpretToI64(const LAllocation& input) + : LWasmReinterpretBase(classOpcode) { + setOperand(0, input); + } +}; + +namespace details { +template <size_t Defs, size_t Ops, size_t Temps> +class RotateBase : public LInstructionHelper<Defs, Ops, Temps> { + typedef LInstructionHelper<Defs, Ops, Temps> Base; + + protected: + explicit RotateBase(LNode::Opcode opcode) : Base(opcode) {} + + public: + MRotate* mir() { return Base::mir_->toRotate(); } +}; +} // namespace details + +class LRotate : public details::RotateBase<1, 2, 0> { + public: + LIR_HEADER(Rotate); + + LRotate() : RotateBase(classOpcode) {} + + const LAllocation* input() { return getOperand(0); } + LAllocation* count() { return getOperand(1); } +}; + +class LRotateI64 + : public details::RotateBase<INT64_PIECES, INT64_PIECES + 1, 1> { + public: + LIR_HEADER(RotateI64); + + LRotateI64() : RotateBase(classOpcode) { + setTemp(0, LDefinition::BogusTemp()); + } + + static const size_t Input = 0; + static const size_t Count = INT64_PIECES; + + const LInt64Allocation input() { return getInt64Operand(Input); } + const LDefinition* temp() { return getTemp(0); } + LAllocation* count() { return getOperand(Count); } +}; + +// Allocate a new arguments object for an inlined frame. +class LCreateInlinedArgumentsObject : public LVariadicInstruction<1, 2> { + public: + LIR_HEADER(CreateInlinedArgumentsObject) + + static const size_t CallObj = 0; + static const size_t Callee = 1; + static const size_t NumNonArgumentOperands = 2; + static size_t ArgIndex(size_t i) { + return NumNonArgumentOperands + BOX_PIECES * i; + } + + LCreateInlinedArgumentsObject(uint32_t numOperands, const LDefinition& temp1, + const LDefinition& temp2) + : LVariadicInstruction(classOpcode, numOperands) { + setIsCall(); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LAllocation* getCallObject() { return getOperand(CallObj); } + const LAllocation* getCallee() { return getOperand(Callee); } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + + MCreateInlinedArgumentsObject* mir() const { + return mir_->toCreateInlinedArgumentsObject(); + } +}; + +class LGetInlinedArgument : public LVariadicInstruction<BOX_PIECES, 0> { + public: + LIR_HEADER(GetInlinedArgument) + + static const size_t Index = 0; + static const size_t NumNonArgumentOperands = 1; + static size_t ArgIndex(size_t i) { + return NumNonArgumentOperands + BOX_PIECES * i; + } + + explicit LGetInlinedArgument(uint32_t numOperands) + : LVariadicInstruction(classOpcode, numOperands) {} + + const LAllocation* getIndex() { return getOperand(Index); } + + MGetInlinedArgument* mir() const { return mir_->toGetInlinedArgument(); } +}; + +class LGetInlinedArgumentHole : public LVariadicInstruction<BOX_PIECES, 0> { + public: + LIR_HEADER(GetInlinedArgumentHole) + + static const size_t Index = 0; + static const size_t NumNonArgumentOperands = 1; + static size_t ArgIndex(size_t i) { + return NumNonArgumentOperands + BOX_PIECES * i; + } + + explicit LGetInlinedArgumentHole(uint32_t numOperands) + : LVariadicInstruction(classOpcode, numOperands) {} + + const LAllocation* getIndex() { return getOperand(Index); } + + MGetInlinedArgumentHole* mir() const { + return mir_->toGetInlinedArgumentHole(); + } +}; + +class LInlineArgumentsSlice : public LVariadicInstruction<1, 1> { + public: + LIR_HEADER(InlineArgumentsSlice) + + static const size_t Begin = 0; + static const size_t Count = 1; + static const size_t NumNonArgumentOperands = 2; + static size_t ArgIndex(size_t i) { + return NumNonArgumentOperands + BOX_PIECES * i; + } + + explicit LInlineArgumentsSlice(uint32_t numOperands, const LDefinition& temp) + : LVariadicInstruction(classOpcode, numOperands) { + setTemp(0, temp); + } + + const LAllocation* begin() { return getOperand(Begin); } + const LAllocation* count() { return getOperand(Count); } + const LDefinition* temp() { return getTemp(0); } + + MInlineArgumentsSlice* mir() const { return mir_->toInlineArgumentsSlice(); } +}; + +// Common code for LIR descended from MCall. +template <size_t Defs, size_t Operands, size_t Temps> +class LJSCallInstructionHelper + : public LCallInstructionHelper<Defs, Operands, Temps> { + protected: + explicit LJSCallInstructionHelper(LNode::Opcode opcode) + : LCallInstructionHelper<Defs, Operands, Temps>(opcode) {} + + public: + MCall* mir() const { return this->mir_->toCall(); } + + bool hasSingleTarget() const { return getSingleTarget() != nullptr; } + WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); } + + // Does not include |this|. + uint32_t numActualArgs() const { return mir()->numActualArgs(); } + + bool isConstructing() const { return mir()->isConstructing(); } + bool ignoresReturnValue() const { return mir()->ignoresReturnValue(); } +}; + +// Generates a polymorphic callsite, wherein the function being called is +// unknown and anticipated to vary. +class LCallGeneric : public LJSCallInstructionHelper<BOX_PIECES, 1, 2> { + public: + LIR_HEADER(CallGeneric) + + LCallGeneric(const LAllocation& func, const LDefinition& nargsreg, + const LDefinition& tmpobjreg) + : LJSCallInstructionHelper(classOpcode) { + setOperand(0, func); + setTemp(0, nargsreg); + setTemp(1, tmpobjreg); + } + + const LAllocation* getFunction() { return getOperand(0); } + const LDefinition* getNargsReg() { return getTemp(0); } + const LDefinition* getTempObject() { return getTemp(1); } +}; + +// Generates a hardcoded callsite for a known, non-native target. +class LCallKnown : public LJSCallInstructionHelper<BOX_PIECES, 1, 1> { + public: + LIR_HEADER(CallKnown) + + LCallKnown(const LAllocation& func, const LDefinition& tmpobjreg) + : LJSCallInstructionHelper(classOpcode) { + setOperand(0, func); + setTemp(0, tmpobjreg); + } + + const LAllocation* getFunction() { return getOperand(0); } + const LDefinition* getTempObject() { return getTemp(0); } +}; + +// Generates a hardcoded callsite for a known, native target. +class LCallNative : public LJSCallInstructionHelper<BOX_PIECES, 0, 4> { + public: + LIR_HEADER(CallNative) + + LCallNative(const LDefinition& argContext, const LDefinition& argUintN, + const LDefinition& argVp, const LDefinition& tmpreg) + : LJSCallInstructionHelper(classOpcode) { + // Registers used for callWithABI(). + setTemp(0, argContext); + setTemp(1, argUintN); + setTemp(2, argVp); + + // Temporary registers. + setTemp(3, tmpreg); + } + + const LDefinition* getArgContextReg() { return getTemp(0); } + const LDefinition* getArgUintNReg() { return getTemp(1); } + const LDefinition* getArgVpReg() { return getTemp(2); } + const LDefinition* getTempReg() { return getTemp(3); } +}; + +class LCallClassHook : public LCallInstructionHelper<BOX_PIECES, 1, 4> { + public: + LIR_HEADER(CallClassHook) + + LCallClassHook(const LAllocation& callee, const LDefinition& argContext, + const LDefinition& argUintN, const LDefinition& argVp, + const LDefinition& tmpreg) + : LCallInstructionHelper(classOpcode) { + setOperand(0, callee); + + // Registers used for callWithABI(). + setTemp(0, argContext); + setTemp(1, argUintN); + setTemp(2, argVp); + + // Temporary registers. + setTemp(3, tmpreg); + } + + MCallClassHook* mir() const { return mir_->toCallClassHook(); } + + const LAllocation* getCallee() { return this->getOperand(0); } + + const LDefinition* getArgContextReg() { return getTemp(0); } + const LDefinition* getArgUintNReg() { return getTemp(1); } + const LDefinition* getArgVpReg() { return getTemp(2); } + const LDefinition* getTempReg() { return getTemp(3); } +}; + +// Generates a hardcoded callsite for a known, DOM-native target. +class LCallDOMNative : public LJSCallInstructionHelper<BOX_PIECES, 0, 4> { + public: + LIR_HEADER(CallDOMNative) + + LCallDOMNative(const LDefinition& argJSContext, const LDefinition& argObj, + const LDefinition& argPrivate, const LDefinition& argArgs) + : LJSCallInstructionHelper(classOpcode) { + setTemp(0, argJSContext); + setTemp(1, argObj); + setTemp(2, argPrivate); + setTemp(3, argArgs); + } + + const LDefinition* getArgJSContext() { return getTemp(0); } + const LDefinition* getArgObj() { return getTemp(1); } + const LDefinition* getArgPrivate() { return getTemp(2); } + const LDefinition* getArgArgs() { return getTemp(3); } +}; + +class LUnreachable : public LControlInstructionHelper<0, 0, 0> { + public: + LIR_HEADER(Unreachable) + + LUnreachable() : LControlInstructionHelper(classOpcode) {} +}; + +class LUnreachableResultV : public LInstructionHelper<BOX_PIECES, 0, 0> { + public: + LIR_HEADER(UnreachableResultV) + + LUnreachableResultV() : LInstructionHelper(classOpcode) {} +}; + +template <size_t defs, size_t ops> +class LDOMPropertyInstructionHelper + : public LCallInstructionHelper<defs, 1 + ops, 3> { + protected: + LDOMPropertyInstructionHelper(LNode::Opcode opcode, + const LDefinition& JSContextReg, + const LAllocation& ObjectReg, + const LDefinition& PrivReg, + const LDefinition& ValueReg) + : LCallInstructionHelper<defs, 1 + ops, 3>(opcode) { + this->setOperand(0, ObjectReg); + this->setTemp(0, JSContextReg); + this->setTemp(1, PrivReg); + this->setTemp(2, ValueReg); + } + + public: + const LDefinition* getJSContextReg() { return this->getTemp(0); } + const LAllocation* getObjectReg() { return this->getOperand(0); } + const LDefinition* getPrivReg() { return this->getTemp(1); } + const LDefinition* getValueReg() { return this->getTemp(2); } +}; + +class LGetDOMProperty : public LDOMPropertyInstructionHelper<BOX_PIECES, 0> { + public: + LIR_HEADER(GetDOMProperty) + + LGetDOMProperty(const LDefinition& JSContextReg, const LAllocation& ObjectReg, + const LDefinition& PrivReg, const LDefinition& ValueReg) + : LDOMPropertyInstructionHelper<BOX_PIECES, 0>( + classOpcode, JSContextReg, ObjectReg, PrivReg, ValueReg) {} + + MGetDOMProperty* mir() const { return mir_->toGetDOMProperty(); } +}; + +class LGetDOMMemberV : public LInstructionHelper<BOX_PIECES, 1, 0> { + public: + LIR_HEADER(GetDOMMemberV); + explicit LGetDOMMemberV(const LAllocation& object) + : LInstructionHelper(classOpcode) { + setOperand(0, object); + } + + const LAllocation* object() { return getOperand(0); } + + MGetDOMMember* mir() const { return mir_->toGetDOMMember(); } +}; + +class LSetDOMProperty : public LDOMPropertyInstructionHelper<0, BOX_PIECES> { + public: + LIR_HEADER(SetDOMProperty) + + LSetDOMProperty(const LDefinition& JSContextReg, const LAllocation& ObjectReg, + const LBoxAllocation& value, const LDefinition& PrivReg, + const LDefinition& ValueReg) + : LDOMPropertyInstructionHelper<0, BOX_PIECES>( + classOpcode, JSContextReg, ObjectReg, PrivReg, ValueReg) { + setBoxOperand(Value, value); + } + + static const size_t Value = 1; + + MSetDOMProperty* mir() const { return mir_->toSetDOMProperty(); } +}; + +// Generates a polymorphic callsite, wherein the function being called is +// unknown and anticipated to vary. +class LApplyArgsGeneric + : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 2, 2> { + public: + LIR_HEADER(ApplyArgsGeneric) + + LApplyArgsGeneric(const LAllocation& func, const LAllocation& argc, + const LBoxAllocation& thisv, const LDefinition& tmpobjreg, + const LDefinition& tmpcopy) + : LCallInstructionHelper(classOpcode) { + setOperand(0, func); + setOperand(1, argc); + setBoxOperand(ThisIndex, thisv); + setTemp(0, tmpobjreg); + setTemp(1, tmpcopy); + } + + MApplyArgs* mir() const { return mir_->toApplyArgs(); } + + bool hasSingleTarget() const { return getSingleTarget() != nullptr; } + WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); } + + uint32_t numExtraFormals() const { return mir()->numExtraFormals(); } + + const LAllocation* getFunction() { return getOperand(0); } + const LAllocation* getArgc() { return getOperand(1); } + static const size_t ThisIndex = 2; + + const LDefinition* getTempObject() { return getTemp(0); } + const LDefinition* getTempForArgCopy() { return getTemp(1); } +}; + +class LApplyArgsObj + : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 2, 2> { + public: + LIR_HEADER(ApplyArgsObj) + + LApplyArgsObj(const LAllocation& func, const LAllocation& argsObj, + const LBoxAllocation& thisv, const LDefinition& tmpObjReg, + const LDefinition& tmpCopy) + : LCallInstructionHelper(classOpcode) { + setOperand(0, func); + setOperand(1, argsObj); + setBoxOperand(ThisIndex, thisv); + setTemp(0, tmpObjReg); + setTemp(1, tmpCopy); + } + + MApplyArgsObj* mir() const { return mir_->toApplyArgsObj(); } + + bool hasSingleTarget() const { return getSingleTarget() != nullptr; } + WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); } + + const LAllocation* getFunction() { return getOperand(0); } + const LAllocation* getArgsObj() { return getOperand(1); } + // All registers are calltemps. argc is mapped to the same register as + // ArgsObj. argc becomes live as ArgsObj is dying. + const LAllocation* getArgc() { return getOperand(1); } + static const size_t ThisIndex = 2; + + const LDefinition* getTempObject() { return getTemp(0); } + const LDefinition* getTempForArgCopy() { return getTemp(1); } +}; + +class LApplyArrayGeneric + : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 2, 2> { + public: + LIR_HEADER(ApplyArrayGeneric) + + LApplyArrayGeneric(const LAllocation& func, const LAllocation& elements, + const LBoxAllocation& thisv, const LDefinition& tmpobjreg, + const LDefinition& tmpcopy) + : LCallInstructionHelper(classOpcode) { + setOperand(0, func); + setOperand(1, elements); + setBoxOperand(ThisIndex, thisv); + setTemp(0, tmpobjreg); + setTemp(1, tmpcopy); + } + + MApplyArray* mir() const { return mir_->toApplyArray(); } + + bool hasSingleTarget() const { return getSingleTarget() != nullptr; } + WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); } + + const LAllocation* getFunction() { return getOperand(0); } + const LAllocation* getElements() { return getOperand(1); } + // argc is mapped to the same register as elements: argc becomes + // live as elements is dying, all registers are calltemps. + const LAllocation* getArgc() { return getOperand(1); } + static const size_t ThisIndex = 2; + + const LDefinition* getTempObject() { return getTemp(0); } + const LDefinition* getTempForArgCopy() { return getTemp(1); } +}; + +class LConstructArgsGeneric + : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 3, 1> { + public: + LIR_HEADER(ConstructArgsGeneric) + + LConstructArgsGeneric(const LAllocation& func, const LAllocation& argc, + const LAllocation& newTarget, + const LBoxAllocation& thisv, + const LDefinition& tmpobjreg) + : LCallInstructionHelper(classOpcode) { + setOperand(0, func); + setOperand(1, argc); + setOperand(2, newTarget); + setBoxOperand(ThisIndex, thisv); + setTemp(0, tmpobjreg); + } + + MConstructArgs* mir() const { return mir_->toConstructArgs(); } + + bool hasSingleTarget() const { return getSingleTarget() != nullptr; } + WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); } + + uint32_t numExtraFormals() const { return mir()->numExtraFormals(); } + + const LAllocation* getFunction() { return getOperand(0); } + const LAllocation* getArgc() { return getOperand(1); } + const LAllocation* getNewTarget() { return getOperand(2); } + + static const size_t ThisIndex = 3; + + const LDefinition* getTempObject() { return getTemp(0); } + + // tempForArgCopy is mapped to the same register as newTarget: + // tempForArgCopy becomes live as newTarget is dying, all registers are + // calltemps. + const LAllocation* getTempForArgCopy() { return getOperand(2); } +}; + +class LConstructArrayGeneric + : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 3, 1> { + public: + LIR_HEADER(ConstructArrayGeneric) + + LConstructArrayGeneric(const LAllocation& func, const LAllocation& elements, + const LAllocation& newTarget, + const LBoxAllocation& thisv, + const LDefinition& tmpobjreg) + : LCallInstructionHelper(classOpcode) { + setOperand(0, func); + setOperand(1, elements); + setOperand(2, newTarget); + setBoxOperand(ThisIndex, thisv); + setTemp(0, tmpobjreg); + } + + MConstructArray* mir() const { return mir_->toConstructArray(); } + + bool hasSingleTarget() const { return getSingleTarget() != nullptr; } + WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); } + + const LAllocation* getFunction() { return getOperand(0); } + const LAllocation* getElements() { return getOperand(1); } + const LAllocation* getNewTarget() { return getOperand(2); } + + static const size_t ThisIndex = 3; + + const LDefinition* getTempObject() { return getTemp(0); } + + // argc is mapped to the same register as elements: argc becomes + // live as elements is dying, all registers are calltemps. + const LAllocation* getArgc() { return getOperand(1); } + + // tempForArgCopy is mapped to the same register as newTarget: + // tempForArgCopy becomes live as newTarget is dying, all registers are + // calltemps. + const LAllocation* getTempForArgCopy() { return getOperand(2); } +}; + +// Takes in either an integer or boolean input and tests it for truthiness. +class LTestIAndBranch : public LControlInstructionHelper<2, 1, 0> { + public: + LIR_HEADER(TestIAndBranch) + + LTestIAndBranch(const LAllocation& in, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode) { + setOperand(0, in); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } +}; + +// Takes in an int64 input and tests it for truthiness. +class LTestI64AndBranch : public LControlInstructionHelper<2, INT64_PIECES, 0> { + public: + LIR_HEADER(TestI64AndBranch) + + LTestI64AndBranch(const LInt64Allocation& in, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode) { + setInt64Operand(0, in); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } +}; + +// Takes in a double input and tests it for truthiness. +class LTestDAndBranch : public LControlInstructionHelper<2, 1, 0> { + public: + LIR_HEADER(TestDAndBranch) + + LTestDAndBranch(const LAllocation& in, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode) { + setOperand(0, in); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } +}; + +// Takes in a float32 input and tests it for truthiness. +class LTestFAndBranch : public LControlInstructionHelper<2, 1, 0> { + public: + LIR_HEADER(TestFAndBranch) + + LTestFAndBranch(const LAllocation& in, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode) { + setOperand(0, in); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } +}; + +// Takes in a bigint input and tests it for truthiness. +class LTestBIAndBranch : public LControlInstructionHelper<2, 1, 0> { + public: + LIR_HEADER(TestBIAndBranch) + + LTestBIAndBranch(const LAllocation& in, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode) { + setOperand(0, in); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + MBasicBlock* ifTrue() { return getSuccessor(0); } + MBasicBlock* ifFalse() { return getSuccessor(1); } +}; + +// Takes an object and tests it for truthiness. An object is falsy iff it +// emulates |undefined|; see js::EmulatesUndefined. +class LTestOAndBranch : public LControlInstructionHelper<2, 1, 1> { + public: + LIR_HEADER(TestOAndBranch) + + LTestOAndBranch(const LAllocation& input, MBasicBlock* ifTruthy, + MBasicBlock* ifFalsy, const LDefinition& temp) + : LControlInstructionHelper(classOpcode) { + setOperand(0, input); + setSuccessor(0, ifTruthy); + setSuccessor(1, ifFalsy); + setTemp(0, temp); + } + + const LDefinition* temp() { return getTemp(0); } + + MBasicBlock* ifTruthy() { return getSuccessor(0); } + MBasicBlock* ifFalsy() { return getSuccessor(1); } + + MTest* mir() { return mir_->toTest(); } +}; + +// Takes in a boxed value and tests it for truthiness. +class LTestVAndBranch : public LControlInstructionHelper<2, BOX_PIECES, 3> { + public: + LIR_HEADER(TestVAndBranch) + + LTestVAndBranch(MBasicBlock* ifTruthy, MBasicBlock* ifFalsy, + const LBoxAllocation& input, const LDefinition& temp0, + const LDefinition& temp1, const LDefinition& temp2) + : LControlInstructionHelper(classOpcode) { + setSuccessor(0, ifTruthy); + setSuccessor(1, ifFalsy); + setBoxOperand(Input, input); + setTemp(0, temp0); + setTemp(1, temp1); + setTemp(2, temp2); + } + + static const size_t Input = 0; + + const LDefinition* tempFloat() { return getTemp(0); } + + const LDefinition* temp1() { return getTemp(1); } + + const LDefinition* temp2() { return getTemp(2); } + + MBasicBlock* ifTruthy() { return getSuccessor(0); } + MBasicBlock* ifFalsy() { return getSuccessor(1); } + + MTest* mir() const { return mir_->toTest(); } +}; + +// Compares two integral values of the same JS type, either integer or object. +// For objects, both operands are in registers. +class LCompare : public LInstructionHelper<1, 2, 0> { + JSOp jsop_; + + public: + LIR_HEADER(Compare) + LCompare(JSOp jsop, const LAllocation& left, const LAllocation& right) + : LInstructionHelper(classOpcode), jsop_(jsop) { + setOperand(0, left); + setOperand(1, right); + } + + JSOp jsop() const { return jsop_; } + const LAllocation* left() { return getOperand(0); } + const LAllocation* right() { return getOperand(1); } + MCompare* mir() { return mir_->toCompare(); } + const char* extraName() const { return CodeName(jsop_); } +}; + +class LCompareI64 : public LInstructionHelper<1, 2 * INT64_PIECES, 0> { + JSOp jsop_; + + public: + LIR_HEADER(CompareI64) + + static const size_t Lhs = 0; + static const size_t Rhs = INT64_PIECES; + + LCompareI64(JSOp jsop, const LInt64Allocation& left, + const LInt64Allocation& right) + : LInstructionHelper(classOpcode), jsop_(jsop) { + setInt64Operand(Lhs, left); + setInt64Operand(Rhs, right); + } + + JSOp jsop() const { return jsop_; } + MCompare* mir() { return mir_->toCompare(); } + const char* extraName() const { return CodeName(jsop_); } +}; + +class LCompareI64AndBranch + : public LControlInstructionHelper<2, 2 * INT64_PIECES, 0> { + MCompare* cmpMir_; + JSOp jsop_; + + public: + LIR_HEADER(CompareI64AndBranch) + + static const size_t Lhs = 0; + static const size_t Rhs = INT64_PIECES; + + LCompareI64AndBranch(MCompare* cmpMir, JSOp jsop, + const LInt64Allocation& left, + const LInt64Allocation& right, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir), jsop_(jsop) { + setInt64Operand(Lhs, left); + setInt64Operand(Rhs, right); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + JSOp jsop() const { return jsop_; } + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + MTest* mir() const { return mir_->toTest(); } + MCompare* cmpMir() const { return cmpMir_; } + const char* extraName() const { return CodeName(jsop_); } +}; + +// Compares two integral values of the same JS type, either integer or object. +// For objects, both operands are in registers. +class LCompareAndBranch : public LControlInstructionHelper<2, 2, 0> { + MCompare* cmpMir_; + JSOp jsop_; + + public: + LIR_HEADER(CompareAndBranch) + LCompareAndBranch(MCompare* cmpMir, JSOp jsop, const LAllocation& left, + const LAllocation& right, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir), jsop_(jsop) { + setOperand(0, left); + setOperand(1, right); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + JSOp jsop() const { return jsop_; } + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + const LAllocation* left() { return getOperand(0); } + const LAllocation* right() { return getOperand(1); } + MTest* mir() const { return mir_->toTest(); } + MCompare* cmpMir() const { return cmpMir_; } + const char* extraName() const { return CodeName(jsop_); } +}; + +class LCompareDAndBranch : public LControlInstructionHelper<2, 2, 0> { + MCompare* cmpMir_; + + public: + LIR_HEADER(CompareDAndBranch) + + LCompareDAndBranch(MCompare* cmpMir, const LAllocation& left, + const LAllocation& right, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) { + setOperand(0, left); + setOperand(1, right); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + const LAllocation* left() { return getOperand(0); } + const LAllocation* right() { return getOperand(1); } + MTest* mir() const { return mir_->toTest(); } + MCompare* cmpMir() const { return cmpMir_; } +}; + +class LCompareFAndBranch : public LControlInstructionHelper<2, 2, 0> { + MCompare* cmpMir_; + + public: + LIR_HEADER(CompareFAndBranch) + LCompareFAndBranch(MCompare* cmpMir, const LAllocation& left, + const LAllocation& right, MBasicBlock* ifTrue, + MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) { + setOperand(0, left); + setOperand(1, right); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + const LAllocation* left() { return getOperand(0); } + const LAllocation* right() { return getOperand(1); } + MTest* mir() const { return mir_->toTest(); } + MCompare* cmpMir() const { return cmpMir_; } +}; + +class LBitAndAndBranch : public LControlInstructionHelper<2, 2, 0> { + // This denotes only a single-word AND on the target. Hence `is64_` is + // required to be `false` on a 32-bit target. + bool is64_; + Assembler::Condition cond_; + + public: + LIR_HEADER(BitAndAndBranch) + LBitAndAndBranch(MBasicBlock* ifTrue, MBasicBlock* ifFalse, bool is64, + Assembler::Condition cond = Assembler::NonZero) + : LControlInstructionHelper(classOpcode), is64_(is64), cond_(cond) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + const LAllocation* left() { return getOperand(0); } + const LAllocation* right() { return getOperand(1); } + bool is64() const { return is64_; } + Assembler::Condition cond() const { + MOZ_ASSERT(cond_ == Assembler::Zero || cond_ == Assembler::NonZero); + return cond_; + } +}; + +class LIsNullOrLikeUndefinedAndBranchV + : public LControlInstructionHelper<2, BOX_PIECES, 2> { + MCompare* cmpMir_; + + public: + LIR_HEADER(IsNullOrLikeUndefinedAndBranchV) + + LIsNullOrLikeUndefinedAndBranchV(MCompare* cmpMir, MBasicBlock* ifTrue, + MBasicBlock* ifFalse, + const LBoxAllocation& value, + const LDefinition& temp, + const LDefinition& tempToUnbox) + : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setBoxOperand(Value, value); + setTemp(0, temp); + setTemp(1, tempToUnbox); + } + + static const size_t Value = 0; + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + MTest* mir() const { return mir_->toTest(); } + MCompare* cmpMir() const { return cmpMir_; } + const LDefinition* temp() { return getTemp(0); } + const LDefinition* tempToUnbox() { return getTemp(1); } +}; + +class LIsNullOrLikeUndefinedAndBranchT + : public LControlInstructionHelper<2, 1, 1> { + MCompare* cmpMir_; + + public: + LIR_HEADER(IsNullOrLikeUndefinedAndBranchT) + + LIsNullOrLikeUndefinedAndBranchT(MCompare* cmpMir, const LAllocation& input, + MBasicBlock* ifTrue, MBasicBlock* ifFalse, + const LDefinition& temp) + : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) { + setOperand(0, input); + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setTemp(0, temp); + } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + MTest* mir() const { return mir_->toTest(); } + MCompare* cmpMir() const { return cmpMir_; } + const LDefinition* temp() { return getTemp(0); } +}; + +class LIsNullAndBranch : public LControlInstructionHelper<2, BOX_PIECES, 0> { + MCompare* cmpMir_; + + public: + LIR_HEADER(IsNullAndBranch) + + LIsNullAndBranch(MCompare* cmpMir, MBasicBlock* ifTrue, MBasicBlock* ifFalse, + const LBoxAllocation& value) + : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setBoxOperand(Value, value); + } + + static const size_t Value = 0; + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + MTest* mir() const { return mir_->toTest(); } + MCompare* cmpMir() const { return cmpMir_; } +}; + +class LIsUndefinedAndBranch + : public LControlInstructionHelper<2, BOX_PIECES, 0> { + MCompare* cmpMir_; + + public: + LIR_HEADER(IsUndefinedAndBranch) + + LIsUndefinedAndBranch(MCompare* cmpMir, MBasicBlock* ifTrue, + MBasicBlock* ifFalse, const LBoxAllocation& value) + : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setBoxOperand(Value, value); + } + + static const size_t Value = 0; + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + MTest* mir() const { return mir_->toTest(); } + MCompare* cmpMir() const { return cmpMir_; } +}; + +// Bitwise not operation, takes a 32-bit integer as input and returning +// a 32-bit integer result as an output. +class LBitNotI : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(BitNotI) + + LBitNotI() : LInstructionHelper(classOpcode) {} +}; + +class LBitNotI64 : public LInstructionHelper<INT64_PIECES, INT64_PIECES, 0> { + public: + LIR_HEADER(BitNotI64) + + LBitNotI64() : LInstructionHelper(classOpcode) {} +}; + +// Binary bitwise operation, taking two 32-bit integers as inputs and returning +// a 32-bit integer result as an output. +class LBitOpI : public LInstructionHelper<1, 2, 0> { + JSOp op_; + + public: + LIR_HEADER(BitOpI) + + explicit LBitOpI(JSOp op) : LInstructionHelper(classOpcode), op_(op) {} + + const char* extraName() const { + if (bitop() == JSOp::Ursh && mir_->toUrsh()->bailoutsDisabled()) { + return "ursh:BailoutsDisabled"; + } + return CodeName(op_); + } + + JSOp bitop() const { return op_; } +}; + +class LBitOpI64 : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0> { + JSOp op_; + + public: + LIR_HEADER(BitOpI64) + + static const size_t Lhs = 0; + static const size_t Rhs = INT64_PIECES; + + explicit LBitOpI64(JSOp op) : LInstructionHelper(classOpcode), op_(op) {} + + const char* extraName() const { return CodeName(op_); } + + JSOp bitop() const { return op_; } +}; + +// Shift operation, taking two 32-bit integers as inputs and returning +// a 32-bit integer result as an output. +class LShiftI : public LBinaryMath<0> { + JSOp op_; + + public: + LIR_HEADER(ShiftI) + + explicit LShiftI(JSOp op) : LBinaryMath(classOpcode), op_(op) {} + + JSOp bitop() { return op_; } + + MInstruction* mir() { return mir_->toInstruction(); } + + const char* extraName() const { return CodeName(op_); } +}; + +class LShiftI64 : public LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 0> { + JSOp op_; + + public: + LIR_HEADER(ShiftI64) + + explicit LShiftI64(JSOp op) : LInstructionHelper(classOpcode), op_(op) {} + + static const size_t Lhs = 0; + static const size_t Rhs = INT64_PIECES; + + JSOp bitop() { return op_; } + + MInstruction* mir() { return mir_->toInstruction(); } + + const char* extraName() const { return CodeName(op_); } +}; + +class LSignExtendInt64 + : public LInstructionHelper<INT64_PIECES, INT64_PIECES, 0> { + public: + LIR_HEADER(SignExtendInt64) + + explicit LSignExtendInt64(const LInt64Allocation& input) + : LInstructionHelper(classOpcode) { + setInt64Operand(0, input); + } + + const MSignExtendInt64* mir() const { return mir_->toSignExtendInt64(); } + + MSignExtendInt64::Mode mode() const { return mir()->mode(); } +}; + +class LUrshD : public LBinaryMath<1> { + public: + LIR_HEADER(UrshD) + + LUrshD(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp); + } + const LDefinition* temp() { return getTemp(0); } +}; + +// Returns from the function being compiled (not used in inlined frames). The +// input must be a box. +class LReturn : public LInstructionHelper<0, BOX_PIECES, 0> { + bool isGenerator_; + + public: + LIR_HEADER(Return) + + explicit LReturn(bool isGenerator) + : LInstructionHelper(classOpcode), isGenerator_(isGenerator) {} + + bool isGenerator() { return isGenerator_; } +}; + +class LMinMaxBase : public LInstructionHelper<1, 2, 0> { + protected: + LMinMaxBase(LNode::Opcode opcode, const LAllocation& first, + const LAllocation& second) + : LInstructionHelper(opcode) { + setOperand(0, first); + setOperand(1, second); + } + + public: + const LAllocation* first() { return this->getOperand(0); } + const LAllocation* second() { return this->getOperand(1); } + const LDefinition* output() { return this->getDef(0); } + MMinMax* mir() const { return mir_->toMinMax(); } + const char* extraName() const { return mir()->isMax() ? "Max" : "Min"; } +}; + +class LMinMaxI : public LMinMaxBase { + public: + LIR_HEADER(MinMaxI) + LMinMaxI(const LAllocation& first, const LAllocation& second) + : LMinMaxBase(classOpcode, first, second) {} +}; + +class LMinMaxD : public LMinMaxBase { + public: + LIR_HEADER(MinMaxD) + LMinMaxD(const LAllocation& first, const LAllocation& second) + : LMinMaxBase(classOpcode, first, second) {} +}; + +class LMinMaxF : public LMinMaxBase { + public: + LIR_HEADER(MinMaxF) + LMinMaxF(const LAllocation& first, const LAllocation& second) + : LMinMaxBase(classOpcode, first, second) {} +}; + +class LMinMaxArrayI : public LInstructionHelper<1, 1, 3> { + public: + LIR_HEADER(MinMaxArrayI); + LMinMaxArrayI(const LAllocation& array, const LDefinition& temp0, + const LDefinition& temp1, const LDefinition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, array); + setTemp(0, temp0); + setTemp(1, temp1); + setTemp(2, temp2); + } + + const LAllocation* array() { return getOperand(0); } + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + const LDefinition* temp3() { return getTemp(2); } + + bool isMax() const { return mir_->toMinMaxArray()->isMax(); } +}; + +class LMinMaxArrayD : public LInstructionHelper<1, 1, 3> { + public: + LIR_HEADER(MinMaxArrayD); + LMinMaxArrayD(const LAllocation& array, const LDefinition& floatTemp, + const LDefinition& temp1, const LDefinition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, array); + setTemp(0, floatTemp); + setTemp(1, temp1); + setTemp(2, temp2); + } + + const LAllocation* array() { return getOperand(0); } + const LDefinition* floatTemp() { return getTemp(0); } + const LDefinition* temp1() { return getTemp(1); } + const LDefinition* temp2() { return getTemp(2); } + + bool isMax() const { return mir_->toMinMaxArray()->isMax(); } +}; + +// Copysign for doubles. +class LCopySignD : public LInstructionHelper<1, 2, 2> { + public: + LIR_HEADER(CopySignD) + explicit LCopySignD() : LInstructionHelper(classOpcode) {} +}; + +// Copysign for float32. +class LCopySignF : public LInstructionHelper<1, 2, 2> { + public: + LIR_HEADER(CopySignF) + explicit LCopySignF() : LInstructionHelper(classOpcode) {} +}; + +class LAtan2D : public LCallInstructionHelper<1, 2, 0> { + public: + LIR_HEADER(Atan2D) + LAtan2D(const LAllocation& y, const LAllocation& x) + : LCallInstructionHelper(classOpcode) { + setOperand(0, y); + setOperand(1, x); + } + + const LAllocation* y() { return getOperand(0); } + + const LAllocation* x() { return getOperand(1); } + + const LDefinition* output() { return getDef(0); } +}; + +class LHypot : public LCallInstructionHelper<1, 4, 0> { + uint32_t numOperands_; + + public: + LIR_HEADER(Hypot) + LHypot(const LAllocation& x, const LAllocation& y) + : LCallInstructionHelper(classOpcode), numOperands_(2) { + setOperand(0, x); + setOperand(1, y); + } + + LHypot(const LAllocation& x, const LAllocation& y, const LAllocation& z) + : LCallInstructionHelper(classOpcode), numOperands_(3) { + setOperand(0, x); + setOperand(1, y); + setOperand(2, z); + } + + LHypot(const LAllocation& x, const LAllocation& y, const LAllocation& z, + const LAllocation& w) + : LCallInstructionHelper(classOpcode), numOperands_(4) { + setOperand(0, x); + setOperand(1, y); + setOperand(2, z); + setOperand(3, w); + } + + uint32_t numArgs() const { return numOperands_; } + + const LAllocation* x() { return getOperand(0); } + + const LAllocation* y() { return getOperand(1); } + + const LDefinition* output() { return getDef(0); } +}; + +class LMathFunctionD : public LCallInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(MathFunctionD) + explicit LMathFunctionD(const LAllocation& input) + : LCallInstructionHelper(classOpcode) { + setOperand(0, input); + } + + MMathFunction* mir() const { return mir_->toMathFunction(); } + const char* extraName() const { + return MMathFunction::FunctionName(mir()->function()); + } +}; + +class LMathFunctionF : public LCallInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(MathFunctionF) + explicit LMathFunctionF(const LAllocation& input) + : LCallInstructionHelper(classOpcode) { + setOperand(0, input); + } + + MMathFunction* mir() const { return mir_->toMathFunction(); } + const char* extraName() const { + return MMathFunction::FunctionName(mir()->function()); + } +}; + +// Adds two integers, returning an integer value. +class LAddI : public LBinaryMath<0> { + bool recoversInput_; + + public: + LIR_HEADER(AddI) + + LAddI() : LBinaryMath(classOpcode), recoversInput_(false) {} + + const char* extraName() const { + return snapshot() ? "OverflowCheck" : nullptr; + } + + bool recoversInput() const { return recoversInput_; } + void setRecoversInput() { recoversInput_ = true; } + + MAdd* mir() const { return mir_->toAdd(); } +}; + +class LAddI64 : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0> { + public: + LIR_HEADER(AddI64) + + LAddI64() : LInstructionHelper(classOpcode) {} + + static const size_t Lhs = 0; + static const size_t Rhs = INT64_PIECES; +}; + +// Subtracts two integers, returning an integer value. +class LSubI : public LBinaryMath<0> { + bool recoversInput_; + + public: + LIR_HEADER(SubI) + + LSubI() : LBinaryMath(classOpcode), recoversInput_(false) {} + + const char* extraName() const { + return snapshot() ? "OverflowCheck" : nullptr; + } + + bool recoversInput() const { return recoversInput_; } + void setRecoversInput() { recoversInput_ = true; } + MSub* mir() const { return mir_->toSub(); } +}; + +inline bool LNode::recoversInput() const { + switch (op()) { + case Opcode::AddI: + return toAddI()->recoversInput(); + case Opcode::SubI: + return toSubI()->recoversInput(); + default: + return false; + } +} + +class LSubI64 : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0> { + public: + LIR_HEADER(SubI64) + + static const size_t Lhs = 0; + static const size_t Rhs = INT64_PIECES; + + LSubI64() : LInstructionHelper(classOpcode) {} +}; + +class LMulI64 : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 1> { + public: + LIR_HEADER(MulI64) + + explicit LMulI64() : LInstructionHelper(classOpcode) { + setTemp(0, LDefinition()); + } + + const LDefinition* temp() { return getTemp(0); } + + static const size_t Lhs = 0; + static const size_t Rhs = INT64_PIECES; +}; + +// Performs an add, sub, mul, or div on two double values. +class LMathD : public LBinaryMath<0> { + JSOp jsop_; + + public: + LIR_HEADER(MathD) + + explicit LMathD(JSOp jsop) : LBinaryMath(classOpcode), jsop_(jsop) {} + + JSOp jsop() const { return jsop_; } + + const char* extraName() const { return CodeName(jsop_); } +}; + +// Performs an add, sub, mul, or div on two double values. +class LMathF : public LBinaryMath<0> { + JSOp jsop_; + + public: + LIR_HEADER(MathF) + + explicit LMathF(JSOp jsop) : LBinaryMath(classOpcode), jsop_(jsop) {} + + JSOp jsop() const { return jsop_; } + + const char* extraName() const { return CodeName(jsop_); } +}; + +class LModD : public LBinaryMath<1> { + public: + LIR_HEADER(ModD) + + LModD(const LAllocation& lhs, const LAllocation& rhs) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setIsCall(); + } + MMod* mir() const { return mir_->toMod(); } +}; + +class LModPowTwoD : public LInstructionHelper<1, 1, 0> { + const uint32_t divisor_; + + public: + LIR_HEADER(ModPowTwoD) + + LModPowTwoD(const LAllocation& lhs, uint32_t divisor) + : LInstructionHelper(classOpcode), divisor_(divisor) { + setOperand(0, lhs); + } + + uint32_t divisor() const { return divisor_; } + const LAllocation* lhs() { return getOperand(0); } + MMod* mir() const { return mir_->toMod(); } +}; + +class LBigIntAdd : public LBinaryMath<2> { + public: + LIR_HEADER(BigIntAdd) + + LBigIntAdd(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } +}; + +class LBigIntSub : public LBinaryMath<2> { + public: + LIR_HEADER(BigIntSub) + + LBigIntSub(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } +}; + +class LBigIntMul : public LBinaryMath<2> { + public: + LIR_HEADER(BigIntMul) + + LBigIntMul(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } +}; + +class LBigIntDiv : public LBinaryMath<2> { + public: + LIR_HEADER(BigIntDiv) + + LBigIntDiv(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + + const MBigIntDiv* mir() const { return mirRaw()->toBigIntDiv(); } +}; + +class LBigIntMod : public LBinaryMath<2> { + public: + LIR_HEADER(BigIntMod) + + LBigIntMod(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + + const MBigIntMod* mir() const { return mirRaw()->toBigIntMod(); } +}; + +class LBigIntPow : public LBinaryMath<2> { + public: + LIR_HEADER(BigIntPow) + + LBigIntPow(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + + const MBigIntPow* mir() const { return mirRaw()->toBigIntPow(); } +}; + +class LBigIntBitAnd : public LBinaryMath<2> { + public: + LIR_HEADER(BigIntBitAnd) + + LBigIntBitAnd(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } +}; + +class LBigIntBitOr : public LBinaryMath<2> { + public: + LIR_HEADER(BigIntBitOr) + + LBigIntBitOr(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } +}; + +class LBigIntBitXor : public LBinaryMath<2> { + public: + LIR_HEADER(BigIntBitXor) + + LBigIntBitXor(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } +}; + +class LBigIntLsh : public LBinaryMath<3> { + public: + LIR_HEADER(BigIntLsh) + + LBigIntLsh(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2, + const LDefinition& temp3) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + setTemp(2, temp3); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + const LDefinition* temp3() { return getTemp(2); } +}; + +class LBigIntRsh : public LBinaryMath<3> { + public: + LIR_HEADER(BigIntRsh) + + LBigIntRsh(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp1, const LDefinition& temp2, + const LDefinition& temp3) + : LBinaryMath(classOpcode) { + setOperand(0, lhs); + setOperand(1, rhs); + setTemp(0, temp1); + setTemp(1, temp2); + setTemp(2, temp3); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + const LDefinition* temp3() { return getTemp(2); } +}; + +class LBigIntIncrement : public LUnaryMath<2> { + public: + LIR_HEADER(BigIntIncrement) + + LBigIntIncrement(const LAllocation& input, const LDefinition& temp1, + const LDefinition& temp2) + : LUnaryMath(classOpcode) { + setOperand(0, input); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } +}; + +class LBigIntDecrement : public LUnaryMath<2> { + public: + LIR_HEADER(BigIntDecrement) + + LBigIntDecrement(const LAllocation& input, const LDefinition& temp1, + const LDefinition& temp2) + : LUnaryMath(classOpcode) { + setOperand(0, input); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } +}; + +class LBigIntNegate : public LUnaryMath<1> { + public: + LIR_HEADER(BigIntNegate) + + LBigIntNegate(const LAllocation& input, const LDefinition& temp) + : LUnaryMath(classOpcode) { + setOperand(0, input); + setTemp(0, temp); + } + + const LDefinition* temp() { return getTemp(0); } +}; + +class LBigIntBitNot : public LUnaryMath<2> { + public: + LIR_HEADER(BigIntBitNot) + + LBigIntBitNot(const LAllocation& input, const LDefinition& temp1, + const LDefinition& temp2) + : LUnaryMath(classOpcode) { + setOperand(0, input); + setTemp(0, temp1); + setTemp(1, temp2); + } + + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } +}; + +// Convert a value to an int32. +// Input: components of a Value +// Output: 32-bit integer +// Bailout: undefined, string, object, or non-int32 double +// Temps: one float register, one GP register +// +// This instruction requires a temporary float register. +class LValueToInt32 : public LInstructionHelper<1, BOX_PIECES, 2> { + public: + enum Mode { NORMAL, TRUNCATE }; + + private: + Mode mode_; + + public: + LIR_HEADER(ValueToInt32) + + LValueToInt32(const LBoxAllocation& input, const LDefinition& temp0, + const LDefinition& temp1, Mode mode) + : LInstructionHelper(classOpcode), mode_(mode) { + setBoxOperand(Input, input); + setTemp(0, temp0); + setTemp(1, temp1); + } + + const char* extraName() const { + switch (mode()) { + case NORMAL: + return "Normal"; + case TRUNCATE: + return "Truncate"; + } + MOZ_CRASH("Invalid mode"); + } + + static const size_t Input = 0; + + Mode mode() const { return mode_; } + const LDefinition* tempFloat() { return getTemp(0); } + const LDefinition* temp() { return getTemp(1); } + MToNumberInt32* mirNormal() const { + MOZ_ASSERT(mode_ == NORMAL); + return mir_->toToNumberInt32(); + } + MTruncateToInt32* mirTruncate() const { + MOZ_ASSERT(mode_ == TRUNCATE); + return mir_->toTruncateToInt32(); + } + MInstruction* mir() const { return mir_->toInstruction(); } +}; + +// Double raised to a half power. +class LPowHalfD : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(PowHalfD); + explicit LPowHalfD(const LAllocation& input) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + } + + const LAllocation* input() { return getOperand(0); } + const LDefinition* output() { return getDef(0); } + MPowHalf* mir() const { return mir_->toPowHalf(); } +}; + +// Passed the BaselineFrame address in the OsrFrameReg via the IonOsrTempData +// populated by PrepareOsrTempData. +// +// Forwards this object to the LOsrValues for Value materialization. +class LOsrEntry : public LInstructionHelper<1, 0, 1> { + protected: + Label label_; + uint32_t frameDepth_; + + public: + LIR_HEADER(OsrEntry) + + explicit LOsrEntry(const LDefinition& temp) + : LInstructionHelper(classOpcode), frameDepth_(0) { + setTemp(0, temp); + } + + void setFrameDepth(uint32_t depth) { frameDepth_ = depth; } + uint32_t getFrameDepth() { return frameDepth_; } + Label* label() { return &label_; } + const LDefinition* temp() { return getTemp(0); } +}; + +// Bailout if index + minimum < 0 or index + maximum >= length. +class LBoundsCheckRange : public LInstructionHelper<0, 2, 1> { + public: + LIR_HEADER(BoundsCheckRange) + + LBoundsCheckRange(const LAllocation& index, const LAllocation& length, + const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, index); + setOperand(1, length); + setTemp(0, temp); + } + const MBoundsCheck* mir() const { return mir_->toBoundsCheck(); } + const LAllocation* index() { return getOperand(0); } + const LAllocation* length() { return getOperand(1); } +}; + +// Load a value from a dense array's elements vector. Bail out if it's the hole +// value. +class LLoadElementV : public LInstructionHelper<BOX_PIECES, 2, 0> { + public: + LIR_HEADER(LoadElementV) + + LLoadElementV(const LAllocation& elements, const LAllocation& index) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + } + + const MLoadElement* mir() const { return mir_->toLoadElement(); } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } +}; + +// Load a value from an array's elements vector, loading |undefined| if we hit a +// hole. Bail out if we get a negative index. +class LLoadElementHole : public LInstructionHelper<BOX_PIECES, 3, 0> { + public: + LIR_HEADER(LoadElementHole) + + LLoadElementHole(const LAllocation& elements, const LAllocation& index, + const LAllocation& initLength) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, initLength); + } + + const MLoadElementHole* mir() const { return mir_->toLoadElementHole(); } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* initLength() { return getOperand(2); } +}; + +// Store a boxed value to a dense array's element vector. +class LStoreElementV : public LInstructionHelper<0, 2 + BOX_PIECES, 0> { + public: + LIR_HEADER(StoreElementV) + + LStoreElementV(const LAllocation& elements, const LAllocation& index, + const LBoxAllocation& value) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setBoxOperand(Value, value); + } + + const char* extraName() const { + return mir()->needsHoleCheck() ? "HoleCheck" : nullptr; + } + + static const size_t Value = 2; + + const MStoreElement* mir() const { return mir_->toStoreElement(); } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } +}; + +// Store a typed value to a dense array's elements vector. Compared to +// LStoreElementV, this instruction can store doubles and constants directly, +// and does not store the type tag if the array is monomorphic and known to +// be packed. +class LStoreElementT : public LInstructionHelper<0, 3, 0> { + public: + LIR_HEADER(StoreElementT) + + LStoreElementT(const LAllocation& elements, const LAllocation& index, + const LAllocation& value) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + } + + const char* extraName() const { + return mir()->needsHoleCheck() ? "HoleCheck" : nullptr; + } + + const MStoreElement* mir() const { return mir_->toStoreElement(); } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { return getOperand(2); } +}; + +class LArrayPopShift : public LInstructionHelper<BOX_PIECES, 1, 2> { + public: + LIR_HEADER(ArrayPopShift) + + LArrayPopShift(const LAllocation& object, const LDefinition& temp0, + const LDefinition& temp1) + : LInstructionHelper(classOpcode) { + setOperand(0, object); + setTemp(0, temp0); + setTemp(1, temp1); + } + + const char* extraName() const { + return mir()->mode() == MArrayPopShift::Pop ? "Pop" : "Shift"; + } + + const MArrayPopShift* mir() const { return mir_->toArrayPopShift(); } + const LAllocation* object() { return getOperand(0); } + const LDefinition* temp0() { return getTemp(0); } + const LDefinition* temp1() { return getTemp(1); } +}; + +class LLoadUnboxedBigInt : public LInstructionHelper<1, 2, 1 + INT64_PIECES> { + public: + LIR_HEADER(LoadUnboxedBigInt) + + LLoadUnboxedBigInt(const LAllocation& elements, const LAllocation& index, + const LDefinition& temp, const LInt64Definition& temp64) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setTemp(0, temp); + setInt64Temp(1, temp64); + } + const MLoadUnboxedScalar* mir() const { return mir_->toLoadUnboxedScalar(); } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LDefinition* temp() { return getTemp(0); } + const LInt64Definition temp64() { return getInt64Temp(1); } +}; + +class LLoadDataViewElement : public LInstructionHelper<1, 3, 1 + INT64_PIECES> { + public: + LIR_HEADER(LoadDataViewElement) + + LLoadDataViewElement(const LAllocation& elements, const LAllocation& index, + const LAllocation& littleEndian, const LDefinition& temp, + const LInt64Definition& temp64) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, littleEndian); + setTemp(0, temp); + setInt64Temp(1, temp64); + } + const MLoadDataViewElement* mir() const { + return mir_->toLoadDataViewElement(); + } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* littleEndian() { return getOperand(2); } + const LDefinition* temp() { return getTemp(0); } + const LInt64Definition temp64() { return getInt64Temp(1); } +}; + +class LLoadTypedArrayElementHoleBigInt + : public LInstructionHelper<BOX_PIECES, 2, 1 + INT64_PIECES> { + public: + LIR_HEADER(LoadTypedArrayElementHoleBigInt) + + LLoadTypedArrayElementHoleBigInt(const LAllocation& object, + const LAllocation& index, + const LDefinition& temp, + const LInt64Definition& temp64) + : LInstructionHelper(classOpcode) { + setOperand(0, object); + setOperand(1, index); + setTemp(0, temp); + setInt64Temp(1, temp64); + } + const MLoadTypedArrayElementHole* mir() const { + return mir_->toLoadTypedArrayElementHole(); + } + const LAllocation* object() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LDefinition* temp() { return getTemp(0); } + const LInt64Definition temp64() { return getInt64Temp(1); } +}; + +class LStoreUnboxedBigInt : public LInstructionHelper<0, 3, INT64_PIECES> { + public: + LIR_HEADER(StoreUnboxedBigInt) + + LStoreUnboxedBigInt(const LAllocation& elements, const LAllocation& index, + const LAllocation& value, const LInt64Definition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp); + } + + const MStoreUnboxedScalar* mir() const { + return mir_->toStoreUnboxedScalar(); + } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { return getOperand(2); } + LInt64Definition temp() { return getInt64Temp(0); } +}; + +class LStoreDataViewElement + : public LInstructionHelper<0, 4, 1 + INT64_PIECES> { + public: + LIR_HEADER(StoreDataViewElement) + + LStoreDataViewElement(const LAllocation& elements, const LAllocation& index, + const LAllocation& value, + const LAllocation& littleEndian, + const LDefinition& temp, const LInt64Definition& temp64) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setOperand(3, littleEndian); + setTemp(0, temp); + setInt64Temp(1, temp64); + } + + const MStoreDataViewElement* mir() const { + return mir_->toStoreDataViewElement(); + } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { return getOperand(2); } + const LAllocation* littleEndian() { return getOperand(3); } + const LDefinition* temp() { return getTemp(0); } + const LInt64Definition temp64() { return getInt64Temp(1); } +}; + +class LStoreTypedArrayElementHoleBigInt + : public LInstructionHelper<0, 4, INT64_PIECES> { + public: + LIR_HEADER(StoreTypedArrayElementHoleBigInt) + + LStoreTypedArrayElementHoleBigInt(const LAllocation& elements, + const LAllocation& length, + const LAllocation& index, + const LAllocation& value, + const LInt64Definition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, length); + setOperand(2, index); + setOperand(3, value); + setInt64Temp(0, temp); + } + + const MStoreTypedArrayElementHole* mir() const { + return mir_->toStoreTypedArrayElementHole(); + } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* length() { return getOperand(1); } + const LAllocation* index() { return getOperand(2); } + const LAllocation* value() { return getOperand(3); } + LInt64Definition temp() { return getInt64Temp(0); } +}; + +class LCompareExchangeTypedArrayElement : public LInstructionHelper<1, 4, 4> { + public: + LIR_HEADER(CompareExchangeTypedArrayElement) + + // ARM, ARM64, x86, x64 + LCompareExchangeTypedArrayElement(const LAllocation& elements, + const LAllocation& index, + const LAllocation& oldval, + const LAllocation& newval, + const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, oldval); + setOperand(3, newval); + setTemp(0, temp); + } + // MIPS32, MIPS64 + LCompareExchangeTypedArrayElement( + const LAllocation& elements, const LAllocation& index, + const LAllocation& oldval, const LAllocation& newval, + const LDefinition& temp, const LDefinition& valueTemp, + const LDefinition& offsetTemp, const LDefinition& maskTemp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, oldval); + setOperand(3, newval); + setTemp(0, temp); + setTemp(1, valueTemp); + setTemp(2, offsetTemp); + setTemp(3, maskTemp); + } + + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* oldval() { return getOperand(2); } + const LAllocation* newval() { return getOperand(3); } + const LDefinition* temp() { return getTemp(0); } + + // Temp that may be used on LL/SC platforms for extract/insert bits of word. + const LDefinition* valueTemp() { return getTemp(1); } + const LDefinition* offsetTemp() { return getTemp(2); } + const LDefinition* maskTemp() { return getTemp(3); } + + const MCompareExchangeTypedArrayElement* mir() const { + return mir_->toCompareExchangeTypedArrayElement(); + } +}; + +class LAtomicExchangeTypedArrayElement : public LInstructionHelper<1, 3, 4> { + public: + LIR_HEADER(AtomicExchangeTypedArrayElement) + + // ARM, ARM64, x86, x64 + LAtomicExchangeTypedArrayElement(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setTemp(0, temp); + } + // MIPS32, MIPS64 + LAtomicExchangeTypedArrayElement(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LDefinition& temp, + const LDefinition& valueTemp, + const LDefinition& offsetTemp, + const LDefinition& maskTemp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setTemp(0, temp); + setTemp(1, valueTemp); + setTemp(2, offsetTemp); + setTemp(3, maskTemp); + } + + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { return getOperand(2); } + const LDefinition* temp() { return getTemp(0); } + + // Temp that may be used on LL/SC platforms for extract/insert bits of word. + const LDefinition* valueTemp() { return getTemp(1); } + const LDefinition* offsetTemp() { return getTemp(2); } + const LDefinition* maskTemp() { return getTemp(3); } + + const MAtomicExchangeTypedArrayElement* mir() const { + return mir_->toAtomicExchangeTypedArrayElement(); + } +}; + +class LAtomicTypedArrayElementBinop : public LInstructionHelper<1, 3, 5> { + public: + LIR_HEADER(AtomicTypedArrayElementBinop) + + static const int32_t valueOp = 2; + + // ARM, ARM64, x86, x64 + LAtomicTypedArrayElementBinop(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LDefinition& temp1, + const LDefinition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setTemp(0, temp1); + setTemp(1, temp2); + } + // MIPS32, MIPS64 + LAtomicTypedArrayElementBinop(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LDefinition& temp2, + const LDefinition& valueTemp, + const LDefinition& offsetTemp, + const LDefinition& maskTemp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setTemp(0, LDefinition::BogusTemp()); + setTemp(1, temp2); + setTemp(2, valueTemp); + setTemp(3, offsetTemp); + setTemp(4, maskTemp); + } + + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { + MOZ_ASSERT(valueOp == 2); + return getOperand(2); + } + const LDefinition* temp1() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + + // Temp that may be used on LL/SC platforms for extract/insert bits of word. + const LDefinition* valueTemp() { return getTemp(2); } + const LDefinition* offsetTemp() { return getTemp(3); } + const LDefinition* maskTemp() { return getTemp(4); } + + const MAtomicTypedArrayElementBinop* mir() const { + return mir_->toAtomicTypedArrayElementBinop(); + } +}; + +// Atomic binary operation where the result is discarded. +class LAtomicTypedArrayElementBinopForEffect + : public LInstructionHelper<0, 3, 4> { + public: + LIR_HEADER(AtomicTypedArrayElementBinopForEffect) + + // ARM, ARM64, x86, x64 + LAtomicTypedArrayElementBinopForEffect( + const LAllocation& elements, const LAllocation& index, + const LAllocation& value, + const LDefinition& flagTemp = LDefinition::BogusTemp()) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setTemp(0, flagTemp); + } + // MIPS32, MIPS64 + LAtomicTypedArrayElementBinopForEffect(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LDefinition& valueTemp, + const LDefinition& offsetTemp, + const LDefinition& maskTemp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setTemp(0, LDefinition::BogusTemp()); + setTemp(1, valueTemp); + setTemp(2, offsetTemp); + setTemp(3, maskTemp); + } + + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { return getOperand(2); } + + // Temp that may be used on LL/SC platforms for the flag result of the store. + const LDefinition* flagTemp() { return getTemp(0); } + // Temp that may be used on LL/SC platforms for extract/insert bits of word. + const LDefinition* valueTemp() { return getTemp(1); } + const LDefinition* offsetTemp() { return getTemp(2); } + const LDefinition* maskTemp() { return getTemp(3); } + + const MAtomicTypedArrayElementBinop* mir() const { + return mir_->toAtomicTypedArrayElementBinop(); + } +}; + +class LAtomicLoad64 : public LInstructionHelper<1, 2, 1 + INT64_PIECES> { + public: + LIR_HEADER(AtomicLoad64) + + LAtomicLoad64(const LAllocation& elements, const LAllocation& index, + const LDefinition& temp, const LInt64Definition& temp64) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setTemp(0, temp); + setInt64Temp(1, temp64); + } + const MLoadUnboxedScalar* mir() const { return mir_->toLoadUnboxedScalar(); } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LDefinition* temp() { return getTemp(0); } + LInt64Definition temp64() { return getInt64Temp(1); } +}; + +class LAtomicStore64 : public LInstructionHelper<0, 3, 2 * INT64_PIECES + 1> { + public: + LIR_HEADER(AtomicStore64) + + // x64, ARM64 + LAtomicStore64(const LAllocation& elements, const LAllocation& index, + const LAllocation& value, const LInt64Definition& temp1) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp1); + setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp()); + setTemp(2 * INT64_PIECES, LDefinition::BogusTemp()); + } + + // ARM32 + LAtomicStore64(const LAllocation& elements, const LAllocation& index, + const LAllocation& value, const LInt64Definition& temp1, + const LInt64Definition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp1); + setInt64Temp(INT64_PIECES, temp2); + setTemp(2 * INT64_PIECES, LDefinition::BogusTemp()); + } + + // x86 + LAtomicStore64(const LAllocation& elements, const LAllocation& index, + const LAllocation& value, const LInt64Definition& temp1, + const LDefinition& tempLow) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp1); + setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp()); + setTemp(2 * INT64_PIECES, tempLow); + } + + const MStoreUnboxedScalar* mir() const { + return mir_->toStoreUnboxedScalar(); + } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { return getOperand(2); } + LInt64Definition temp1() { return getInt64Temp(0); } + LInt64Definition temp2() { return getInt64Temp(INT64_PIECES); } + const LDefinition* tempLow() { return getTemp(2 * INT64_PIECES); } +}; + +class LCompareExchangeTypedArrayElement64 + : public LInstructionHelper<1, 4, 3 * INT64_PIECES + 1> { + public: + LIR_HEADER(CompareExchangeTypedArrayElement64) + + // x64, ARM64 + LCompareExchangeTypedArrayElement64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& oldval, + const LAllocation& newval, + const LInt64Definition& temp1, + const LInt64Definition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, oldval); + setOperand(3, newval); + setInt64Temp(0, temp1); + setInt64Temp(INT64_PIECES, temp2); + setInt64Temp(2 * INT64_PIECES, LInt64Definition::BogusTemp()); + setTemp(3 * INT64_PIECES, LDefinition::BogusTemp()); + } + + // x86 + LCompareExchangeTypedArrayElement64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& oldval, + const LAllocation& newval, + const LDefinition& tempLow) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, oldval); + setOperand(3, newval); + setInt64Temp(0, LInt64Definition::BogusTemp()); + setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp()); + setInt64Temp(2 * INT64_PIECES, LInt64Definition::BogusTemp()); + setTemp(3 * INT64_PIECES, tempLow); + } + + // ARM + LCompareExchangeTypedArrayElement64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& oldval, + const LAllocation& newval, + const LInt64Definition& temp1, + const LInt64Definition& temp2, + const LInt64Definition& temp3) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, oldval); + setOperand(3, newval); + setInt64Temp(0, temp1); + setInt64Temp(INT64_PIECES, temp2); + setInt64Temp(2 * INT64_PIECES, temp3); + setTemp(3 * INT64_PIECES, LDefinition::BogusTemp()); + } + + const MCompareExchangeTypedArrayElement* mir() const { + return mir_->toCompareExchangeTypedArrayElement(); + } + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* oldval() { return getOperand(2); } + const LAllocation* newval() { return getOperand(3); } + LInt64Definition temp1() { return getInt64Temp(0); } + LInt64Definition temp2() { return getInt64Temp(INT64_PIECES); } + LInt64Definition temp3() { return getInt64Temp(2 * INT64_PIECES); } + const LDefinition* tempLow() { return getTemp(3 * INT64_PIECES); } +}; + +class LAtomicExchangeTypedArrayElement64 + : public LInstructionHelper<1, 3, INT64_PIECES + 1> { + public: + LIR_HEADER(AtomicExchangeTypedArrayElement64) + + // ARM, ARM64, x64 + LAtomicExchangeTypedArrayElement64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LInt64Definition& temp1, + const LDefinition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp1); + setTemp(INT64_PIECES, temp2); + } + + // x86 + LAtomicExchangeTypedArrayElement64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LInt64Definition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp); + setTemp(INT64_PIECES, LDefinition::BogusTemp()); + } + + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { return getOperand(2); } + LInt64Definition temp1() { return getInt64Temp(0); } + const LDefinition* temp2() { return getTemp(INT64_PIECES); } + + const MAtomicExchangeTypedArrayElement* mir() const { + return mir_->toAtomicExchangeTypedArrayElement(); + } +}; + +class LAtomicTypedArrayElementBinop64 + : public LInstructionHelper<1, 3, 3 * INT64_PIECES> { + public: + LIR_HEADER(AtomicTypedArrayElementBinop64) + + // x86 + LAtomicTypedArrayElementBinop64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LInt64Definition& temp1) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp1); + setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp()); + setInt64Temp(2 * INT64_PIECES, LInt64Definition::BogusTemp()); + } + + // ARM64, x64 + LAtomicTypedArrayElementBinop64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LInt64Definition& temp1, + const LInt64Definition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp1); + setInt64Temp(INT64_PIECES, temp2); + setInt64Temp(2 * INT64_PIECES, LInt64Definition::BogusTemp()); + } + + // ARM + LAtomicTypedArrayElementBinop64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LInt64Definition& temp1, + const LInt64Definition& temp2, + const LInt64Definition& temp3) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp1); + setInt64Temp(INT64_PIECES, temp2); + setInt64Temp(2 * INT64_PIECES, temp3); + } + + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { return getOperand(2); } + LInt64Definition temp1() { return getInt64Temp(0); } + LInt64Definition temp2() { return getInt64Temp(INT64_PIECES); } + LInt64Definition temp3() { return getInt64Temp(2 * INT64_PIECES); } + + const MAtomicTypedArrayElementBinop* mir() const { + return mir_->toAtomicTypedArrayElementBinop(); + } +}; + +// Atomic binary operation where the result is discarded. +class LAtomicTypedArrayElementBinopForEffect64 + : public LInstructionHelper<0, 3, 2 * INT64_PIECES + 1> { + public: + LIR_HEADER(AtomicTypedArrayElementBinopForEffect64) + + // x86 + LAtomicTypedArrayElementBinopForEffect64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LInt64Definition& temp, + const LDefinition& tempLow) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp); + setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp()); + setTemp(2 * INT64_PIECES, tempLow); + } + + // x64 + LAtomicTypedArrayElementBinopForEffect64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LInt64Definition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp); + setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp()); + setTemp(2 * INT64_PIECES, LDefinition::BogusTemp()); + } + + // ARM64 + LAtomicTypedArrayElementBinopForEffect64(const LAllocation& elements, + const LAllocation& index, + const LAllocation& value, + const LInt64Definition& temp1, + const LInt64Definition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, elements); + setOperand(1, index); + setOperand(2, value); + setInt64Temp(0, temp1); + setInt64Temp(INT64_PIECES, temp2); + setTemp(2 * INT64_PIECES, LDefinition::BogusTemp()); + } + + const LAllocation* elements() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + const LAllocation* value() { return getOperand(2); } + LInt64Definition temp1() { return getInt64Temp(0); } + LInt64Definition temp2() { return getInt64Temp(INT64_PIECES); } + const LDefinition* tempLow() { return getTemp(2 * INT64_PIECES); } + + const MAtomicTypedArrayElementBinop* mir() const { + return mir_->toAtomicTypedArrayElementBinop(); + } +}; + +class LIteratorHasIndicesAndBranch : public LControlInstructionHelper<2, 2, 2> { + public: + LIR_HEADER(IteratorHasIndicesAndBranch) + + LIteratorHasIndicesAndBranch(MBasicBlock* ifTrue, MBasicBlock* ifFalse, + const LAllocation& object, + const LAllocation& iterator, + const LDefinition& temp, + const LDefinition& temp2) + : LControlInstructionHelper(classOpcode) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setOperand(0, object); + setOperand(1, iterator); + setTemp(0, temp); + setTemp(1, temp2); + } + + const LAllocation* object() { return getOperand(0); } + const LAllocation* iterator() { return getOperand(1); } + const LDefinition* temp() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } +}; + +class LIsNoIterAndBranch : public LControlInstructionHelper<2, BOX_PIECES, 0> { + public: + LIR_HEADER(IsNoIterAndBranch) + + LIsNoIterAndBranch(MBasicBlock* ifTrue, MBasicBlock* ifFalse, + const LBoxAllocation& input) + : LControlInstructionHelper(classOpcode) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setBoxOperand(Input, input); + } + + static const size_t Input = 0; + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } +}; + +class LInstanceOfCache : public LInstructionHelper<1, BOX_PIECES + 1, 0> { + public: + LIR_HEADER(InstanceOfCache) + LInstanceOfCache(const LBoxAllocation& lhs, const LAllocation& rhs) + : LInstructionHelper(classOpcode) { + setBoxOperand(LHS, lhs); + setOperand(RHS, rhs); + } + + const LDefinition* output() { return this->getDef(0); } + const LAllocation* rhs() { return getOperand(RHS); } + + static const size_t LHS = 0; + static const size_t RHS = BOX_PIECES; +}; + +class LIsObjectAndBranch : public LControlInstructionHelper<2, BOX_PIECES, 0> { + public: + LIR_HEADER(IsObjectAndBranch) + + LIsObjectAndBranch(MBasicBlock* ifTrue, MBasicBlock* ifFalse, + const LBoxAllocation& input) + : LControlInstructionHelper(classOpcode) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setBoxOperand(Input, input); + } + + static const size_t Input = 0; + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } +}; + +class LIsNullOrUndefinedAndBranch + : public LControlInstructionHelper<2, BOX_PIECES, 0> { + MIsNullOrUndefined* isNullOrUndefined_; + + public: + LIR_HEADER(IsNullOrUndefinedAndBranch) + static const size_t Input = 0; + + LIsNullOrUndefinedAndBranch(MIsNullOrUndefined* isNullOrUndefined, + MBasicBlock* ifTrue, MBasicBlock* ifFalse, + const LBoxAllocation& input) + : LControlInstructionHelper(classOpcode), + isNullOrUndefined_(isNullOrUndefined) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setBoxOperand(Input, input); + } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + + MIsNullOrUndefined* isNullOrUndefinedMir() const { + return isNullOrUndefined_; + } +}; + +template <size_t Defs, size_t Ops> +class LWasmSelectBase : public LInstructionHelper<Defs, Ops, 0> { + typedef LInstructionHelper<Defs, Ops, 0> Base; + + protected: + explicit LWasmSelectBase(LNode::Opcode opcode) : Base(opcode) {} + + public: + MWasmSelect* mir() const { return Base::mir_->toWasmSelect(); } +}; + +class LWasmSelect : public LWasmSelectBase<1, 3> { + public: + LIR_HEADER(WasmSelect); + + static const size_t TrueExprIndex = 0; + static const size_t FalseExprIndex = 1; + static const size_t CondIndex = 2; + + LWasmSelect(const LAllocation& trueExpr, const LAllocation& falseExpr, + const LAllocation& cond) + : LWasmSelectBase(classOpcode) { + setOperand(TrueExprIndex, trueExpr); + setOperand(FalseExprIndex, falseExpr); + setOperand(CondIndex, cond); + } + + const LAllocation* trueExpr() { return getOperand(TrueExprIndex); } + const LAllocation* falseExpr() { return getOperand(FalseExprIndex); } + const LAllocation* condExpr() { return getOperand(CondIndex); } +}; + +class LWasmSelectI64 + : public LWasmSelectBase<INT64_PIECES, 2 * INT64_PIECES + 1> { + public: + LIR_HEADER(WasmSelectI64); + + static const size_t TrueExprIndex = 0; + static const size_t FalseExprIndex = INT64_PIECES; + static const size_t CondIndex = INT64_PIECES * 2; + + LWasmSelectI64(const LInt64Allocation& trueExpr, + const LInt64Allocation& falseExpr, const LAllocation& cond) + : LWasmSelectBase(classOpcode) { + setInt64Operand(TrueExprIndex, trueExpr); + setInt64Operand(FalseExprIndex, falseExpr); + setOperand(CondIndex, cond); + } + + const LInt64Allocation trueExpr() { return getInt64Operand(TrueExprIndex); } + const LInt64Allocation falseExpr() { return getInt64Operand(FalseExprIndex); } + const LAllocation* condExpr() { return getOperand(CondIndex); } +}; + +class LWasmCompareAndSelect : public LWasmSelectBase<1, 4> { + MCompare::CompareType compareType_; + JSOp jsop_; + + public: + LIR_HEADER(WasmCompareAndSelect); + + static const size_t LeftExprIndex = 0; + static const size_t RightExprIndex = 1; + static const size_t IfTrueExprIndex = 2; + static const size_t IfFalseExprIndex = 3; + + LWasmCompareAndSelect(const LAllocation& leftExpr, + const LAllocation& rightExpr, + MCompare::CompareType compareType, JSOp jsop, + const LAllocation& ifTrueExpr, + const LAllocation& ifFalseExpr) + : LWasmSelectBase(classOpcode), compareType_(compareType), jsop_(jsop) { + setOperand(LeftExprIndex, leftExpr); + setOperand(RightExprIndex, rightExpr); + setOperand(IfTrueExprIndex, ifTrueExpr); + setOperand(IfFalseExprIndex, ifFalseExpr); + } + + const LAllocation* leftExpr() { return getOperand(LeftExprIndex); } + const LAllocation* rightExpr() { return getOperand(RightExprIndex); } + const LAllocation* ifTrueExpr() { return getOperand(IfTrueExprIndex); } + const LAllocation* ifFalseExpr() { return getOperand(IfFalseExprIndex); } + + MCompare::CompareType compareType() { return compareType_; } + JSOp jsop() { return jsop_; } +}; + +class LWasmBoundsCheck64 + : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0> { + public: + LIR_HEADER(WasmBoundsCheck64); + explicit LWasmBoundsCheck64(const LInt64Allocation& ptr, + const LInt64Allocation& boundsCheckLimit) + : LInstructionHelper(classOpcode) { + setInt64Operand(0, ptr); + setInt64Operand(INT64_PIECES, boundsCheckLimit); + } + MWasmBoundsCheck* mir() const { return mir_->toWasmBoundsCheck(); } + LInt64Allocation ptr() { return getInt64Operand(0); } + LInt64Allocation boundsCheckLimit() { return getInt64Operand(INT64_PIECES); } +}; + +namespace details { + +// This is a base class for LWasmLoad/LWasmLoadI64. +template <size_t Defs, size_t Temp> +class LWasmLoadBase : public LInstructionHelper<Defs, 2, Temp> { + public: + typedef LInstructionHelper<Defs, 2, Temp> Base; + explicit LWasmLoadBase(LNode::Opcode opcode, const LAllocation& ptr, + const LAllocation& memoryBase) + : Base(opcode) { + Base::setOperand(0, ptr); + Base::setOperand(1, memoryBase); + } + MWasmLoad* mir() const { return Base::mir_->toWasmLoad(); } + const LAllocation* ptr() { return Base::getOperand(0); } + const LAllocation* memoryBase() { return Base::getOperand(1); } +}; + +} // namespace details + +class LWasmLoad : public details::LWasmLoadBase<1, 1> { + public: + explicit LWasmLoad(const LAllocation& ptr, + const LAllocation& memoryBase = LAllocation()) + : LWasmLoadBase(classOpcode, ptr, memoryBase) { + setTemp(0, LDefinition::BogusTemp()); + } + + const LDefinition* ptrCopy() { return Base::getTemp(0); } + + LIR_HEADER(WasmLoad); +}; + +class LWasmLoadI64 : public details::LWasmLoadBase<INT64_PIECES, 1> { + public: + explicit LWasmLoadI64(const LAllocation& ptr, + const LAllocation& memoryBase = LAllocation()) + : LWasmLoadBase(classOpcode, ptr, memoryBase) { + setTemp(0, LDefinition::BogusTemp()); + } + + const LDefinition* ptrCopy() { return Base::getTemp(0); } + + LIR_HEADER(WasmLoadI64); +}; + +class LWasmStore : public LInstructionHelper<0, 3, 1> { + public: + LIR_HEADER(WasmStore); + + static const size_t PtrIndex = 0; + static const size_t ValueIndex = 1; + static const size_t MemoryBaseIndex = 2; + + LWasmStore(const LAllocation& ptr, const LAllocation& value, + const LAllocation& memoryBase = LAllocation()) + : LInstructionHelper(classOpcode) { + setOperand(PtrIndex, ptr); + setOperand(ValueIndex, value); + setOperand(MemoryBaseIndex, memoryBase); + setTemp(0, LDefinition::BogusTemp()); + } + MWasmStore* mir() const { return mir_->toWasmStore(); } + const LAllocation* ptr() { return getOperand(PtrIndex); } + const LDefinition* ptrCopy() { return getTemp(0); } + const LAllocation* value() { return getOperand(ValueIndex); } + const LAllocation* memoryBase() { return getOperand(MemoryBaseIndex); } +}; + +class LWasmStoreI64 : public LInstructionHelper<0, INT64_PIECES + 2, 1> { + public: + LIR_HEADER(WasmStoreI64); + + static const size_t PtrIndex = 0; + static const size_t MemoryBaseIndex = 1; + static const size_t ValueIndex = 2; + + LWasmStoreI64(const LAllocation& ptr, const LInt64Allocation& value, + const LAllocation& memoryBase = LAllocation()) + : LInstructionHelper(classOpcode) { + setOperand(PtrIndex, ptr); + setOperand(MemoryBaseIndex, memoryBase); + setInt64Operand(ValueIndex, value); + setTemp(0, LDefinition::BogusTemp()); + } + MWasmStore* mir() const { return mir_->toWasmStore(); } + const LAllocation* ptr() { return getOperand(PtrIndex); } + const LAllocation* memoryBase() { return getOperand(MemoryBaseIndex); } + const LDefinition* ptrCopy() { return getTemp(0); } + const LInt64Allocation value() { return getInt64Operand(ValueIndex); } +}; + +class LWasmCompareExchangeHeap : public LInstructionHelper<1, 4, 4> { + public: + LIR_HEADER(WasmCompareExchangeHeap); + + // ARM, ARM64, x86, x64 + LWasmCompareExchangeHeap(const LAllocation& ptr, const LAllocation& oldValue, + const LAllocation& newValue, + const LAllocation& memoryBase = LAllocation()) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, oldValue); + setOperand(2, newValue); + setOperand(3, memoryBase); + setTemp(0, LDefinition::BogusTemp()); + } + // MIPS32, MIPS64 + LWasmCompareExchangeHeap(const LAllocation& ptr, const LAllocation& oldValue, + const LAllocation& newValue, + const LDefinition& valueTemp, + const LDefinition& offsetTemp, + const LDefinition& maskTemp) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, oldValue); + setOperand(2, newValue); + setOperand(3, LAllocation()); + setTemp(0, LDefinition::BogusTemp()); + setTemp(1, valueTemp); + setTemp(2, offsetTemp); + setTemp(3, maskTemp); + } + + const LAllocation* ptr() { return getOperand(0); } + const LAllocation* oldValue() { return getOperand(1); } + const LAllocation* newValue() { return getOperand(2); } + const LAllocation* memoryBase() { return getOperand(3); } + const LDefinition* addrTemp() { return getTemp(0); } + + void setAddrTemp(const LDefinition& addrTemp) { setTemp(0, addrTemp); } + + // Temp that may be used on LL/SC platforms for extract/insert bits of word. + const LDefinition* valueTemp() { return getTemp(1); } + const LDefinition* offsetTemp() { return getTemp(2); } + const LDefinition* maskTemp() { return getTemp(3); } + + MWasmCompareExchangeHeap* mir() const { + return mir_->toWasmCompareExchangeHeap(); + } +}; + +class LWasmAtomicExchangeHeap : public LInstructionHelper<1, 3, 4> { + public: + LIR_HEADER(WasmAtomicExchangeHeap); + + // ARM, ARM64, x86, x64 + LWasmAtomicExchangeHeap(const LAllocation& ptr, const LAllocation& value, + const LAllocation& memoryBase = LAllocation()) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, value); + setOperand(2, memoryBase); + setTemp(0, LDefinition::BogusTemp()); + } + // MIPS32, MIPS64 + LWasmAtomicExchangeHeap(const LAllocation& ptr, const LAllocation& value, + const LDefinition& valueTemp, + const LDefinition& offsetTemp, + const LDefinition& maskTemp) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, value); + setOperand(2, LAllocation()); + setTemp(0, LDefinition::BogusTemp()); + setTemp(1, valueTemp); + setTemp(2, offsetTemp); + setTemp(3, maskTemp); + } + + const LAllocation* ptr() { return getOperand(0); } + const LAllocation* value() { return getOperand(1); } + const LAllocation* memoryBase() { return getOperand(2); } + const LDefinition* addrTemp() { return getTemp(0); } + + void setAddrTemp(const LDefinition& addrTemp) { setTemp(0, addrTemp); } + + // Temp that may be used on LL/SC platforms for extract/insert bits of word. + const LDefinition* valueTemp() { return getTemp(1); } + const LDefinition* offsetTemp() { return getTemp(2); } + const LDefinition* maskTemp() { return getTemp(3); } + + MWasmAtomicExchangeHeap* mir() const { + return mir_->toWasmAtomicExchangeHeap(); + } +}; + +class LWasmAtomicBinopHeap : public LInstructionHelper<1, 3, 6> { + public: + LIR_HEADER(WasmAtomicBinopHeap); + + static const int32_t valueOp = 1; + + // ARM, ARM64, x86, x64 + LWasmAtomicBinopHeap(const LAllocation& ptr, const LAllocation& value, + const LDefinition& temp, + const LDefinition& flagTemp = LDefinition::BogusTemp(), + const LAllocation& memoryBase = LAllocation()) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, value); + setOperand(2, memoryBase); + setTemp(0, temp); + setTemp(1, LDefinition::BogusTemp()); + setTemp(2, flagTemp); + } + // MIPS32, MIPS64 + LWasmAtomicBinopHeap(const LAllocation& ptr, const LAllocation& value, + const LDefinition& valueTemp, + const LDefinition& offsetTemp, + const LDefinition& maskTemp) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, value); + setOperand(2, LAllocation()); + setTemp(0, LDefinition::BogusTemp()); + setTemp(1, LDefinition::BogusTemp()); + setTemp(2, LDefinition::BogusTemp()); + setTemp(3, valueTemp); + setTemp(4, offsetTemp); + setTemp(5, maskTemp); + } + const LAllocation* ptr() { return getOperand(0); } + const LAllocation* value() { + MOZ_ASSERT(valueOp == 1); + return getOperand(1); + } + const LAllocation* memoryBase() { return getOperand(2); } + const LDefinition* temp() { return getTemp(0); } + + // Temp that may be used on some platforms to hold a computed address. + const LDefinition* addrTemp() { return getTemp(1); } + void setAddrTemp(const LDefinition& addrTemp) { setTemp(1, addrTemp); } + + // Temp that may be used on LL/SC platforms for the flag result of the store. + const LDefinition* flagTemp() { return getTemp(2); } + // Temp that may be used on LL/SC platforms for extract/insert bits of word. + const LDefinition* valueTemp() { return getTemp(3); } + const LDefinition* offsetTemp() { return getTemp(4); } + const LDefinition* maskTemp() { return getTemp(5); } + + MWasmAtomicBinopHeap* mir() const { return mir_->toWasmAtomicBinopHeap(); } +}; + +// Atomic binary operation where the result is discarded. +class LWasmAtomicBinopHeapForEffect : public LInstructionHelper<0, 3, 5> { + public: + LIR_HEADER(WasmAtomicBinopHeapForEffect); + // ARM, ARM64, x86, x64 + LWasmAtomicBinopHeapForEffect( + const LAllocation& ptr, const LAllocation& value, + const LDefinition& flagTemp = LDefinition::BogusTemp(), + const LAllocation& memoryBase = LAllocation()) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, value); + setOperand(2, memoryBase); + setTemp(0, LDefinition::BogusTemp()); + setTemp(1, flagTemp); + } + // MIPS32, MIPS64 + LWasmAtomicBinopHeapForEffect(const LAllocation& ptr, + const LAllocation& value, + const LDefinition& valueTemp, + const LDefinition& offsetTemp, + const LDefinition& maskTemp) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, value); + setOperand(2, LAllocation()); + setTemp(0, LDefinition::BogusTemp()); + setTemp(1, LDefinition::BogusTemp()); + setTemp(2, valueTemp); + setTemp(3, offsetTemp); + setTemp(4, maskTemp); + } + const LAllocation* ptr() { return getOperand(0); } + const LAllocation* value() { return getOperand(1); } + const LAllocation* memoryBase() { return getOperand(2); } + + // Temp that may be used on some platforms to hold a computed address. + const LDefinition* addrTemp() { return getTemp(0); } + void setAddrTemp(const LDefinition& addrTemp) { setTemp(0, addrTemp); } + + // Temp that may be used on LL/SC platforms for the flag result of the store. + const LDefinition* flagTemp() { return getTemp(1); } + // Temp that may be used on LL/SC platforms for extract/insert bits of word. + const LDefinition* valueTemp() { return getTemp(2); } + const LDefinition* offsetTemp() { return getTemp(3); } + const LDefinition* maskTemp() { return getTemp(4); } + + MWasmAtomicBinopHeap* mir() const { return mir_->toWasmAtomicBinopHeap(); } +}; + +class LWasmDerivedPointer : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(WasmDerivedPointer); + explicit LWasmDerivedPointer(const LAllocation& base) + : LInstructionHelper(classOpcode) { + setOperand(0, base); + } + const LAllocation* base() { return getOperand(0); } + uint32_t offset() { return mirRaw()->toWasmDerivedPointer()->offset(); } +}; + +class LWasmDerivedIndexPointer : public LInstructionHelper<1, 2, 0> { + public: + LIR_HEADER(WasmDerivedIndexPointer); + explicit LWasmDerivedIndexPointer(const LAllocation& base, + const LAllocation& index) + : LInstructionHelper(classOpcode) { + setOperand(0, base); + setOperand(1, index); + } + const LAllocation* base() { return getOperand(0); } + const LAllocation* index() { return getOperand(1); } + Scale scale() { return mirRaw()->toWasmDerivedIndexPointer()->scale(); } +}; + +class LWasmParameterI64 : public LInstructionHelper<INT64_PIECES, 0, 0> { + public: + LIR_HEADER(WasmParameterI64); + + LWasmParameterI64() : LInstructionHelper(classOpcode) {} +}; + +// This is used only with LWasmCall. +class LWasmCallIndirectAdjunctSafepoint : public LInstructionHelper<0, 0, 0> { + CodeOffset offs_; + uint32_t framePushedAtStackMapBase_; + + public: + LIR_HEADER(WasmCallIndirectAdjunctSafepoint); + + LWasmCallIndirectAdjunctSafepoint() + : LInstructionHelper(classOpcode), + offs_(0), + framePushedAtStackMapBase_(0) {} + + CodeOffset safepointLocation() const { + MOZ_ASSERT(offs_.offset() != 0); + return offs_; + } + uint32_t framePushedAtStackMapBase() const { + MOZ_ASSERT(offs_.offset() != 0); + return framePushedAtStackMapBase_; + } + void recordSafepointInfo(CodeOffset offs, uint32_t framePushed) { + offs_ = offs; + framePushedAtStackMapBase_ = framePushed; + } +}; + +// LWasmCall may be generated into two function calls in the case of +// call_indirect, one for the fast path and one for the slow path. In that +// case, the node carries a pointer to a companion node, the "adjunct +// safepoint", representing the safepoint for the second of the two calls. The +// dual-call construction is only meaningful for wasm because wasm has no +// invalidation of code; this is not a pattern to be used generally. +class LWasmCall : public LVariadicInstruction<0, 0> { + bool needsBoundsCheck_; + mozilla::Maybe<uint32_t> tableSize_; + LWasmCallIndirectAdjunctSafepoint* adjunctSafepoint_; + + public: + LIR_HEADER(WasmCall); + + LWasmCall(uint32_t numOperands, bool needsBoundsCheck, + mozilla::Maybe<uint32_t> tableSize = mozilla::Nothing()) + : LVariadicInstruction(classOpcode, numOperands), + needsBoundsCheck_(needsBoundsCheck), + tableSize_(tableSize), + adjunctSafepoint_(nullptr) { + this->setIsCall(); + } + + MWasmCallBase* callBase() const { + if (isCatchable()) { + return static_cast<MWasmCallBase*>(mirCatchable()); + } + return static_cast<MWasmCallBase*>(mirUncatchable()); + } + bool isCatchable() const { return mir_->isWasmCallCatchable(); } + MWasmCallCatchable* mirCatchable() const { + return mir_->toWasmCallCatchable(); + } + MWasmCallUncatchable* mirUncatchable() const { + return mir_->toWasmCallUncatchable(); + } + + static bool isCallPreserved(AnyRegister reg) { + // All MWasmCalls preserve the TLS register: + // - internal/indirect calls do by the internal wasm ABI + // - import calls do by explicitly saving/restoring at the callsite + // - builtin calls do because the TLS reg is non-volatile + // See also CodeGeneratorShared::emitWasmCall. + // + // All other registers are not preserved. This is is relied upon by + // MWasmCallCatchable which needs all live registers to be spilled before + // a call. + return !reg.isFloat() && reg.gpr() == InstanceReg; + } + + bool needsBoundsCheck() const { return needsBoundsCheck_; } + mozilla::Maybe<uint32_t> tableSize() const { return tableSize_; } + LWasmCallIndirectAdjunctSafepoint* adjunctSafepoint() const { + MOZ_ASSERT(adjunctSafepoint_ != nullptr); + return adjunctSafepoint_; + } + void setAdjunctSafepoint(LWasmCallIndirectAdjunctSafepoint* asp) { + adjunctSafepoint_ = asp; + } +}; + +class LWasmRegisterResult : public LInstructionHelper<1, 0, 0> { + public: + LIR_HEADER(WasmRegisterResult); + + LWasmRegisterResult() : LInstructionHelper(classOpcode) {} + + MWasmRegisterResult* mir() const { + if (!mir_->isWasmRegisterResult()) { + return nullptr; + } + return mir_->toWasmRegisterResult(); + } +}; + +class LWasmRegisterPairResult : public LInstructionHelper<2, 0, 0> { + public: + LIR_HEADER(WasmRegisterPairResult); + + LWasmRegisterPairResult() : LInstructionHelper(classOpcode) {} + + MDefinition* mir() const { return mirRaw(); } +}; + +inline uint32_t LStackArea::base() const { + return ins()->toWasmStackResultArea()->mir()->base(); +} +inline void LStackArea::setBase(uint32_t base) { + ins()->toWasmStackResultArea()->mir()->setBase(base); +} +inline uint32_t LStackArea::size() const { + return ins()->toWasmStackResultArea()->mir()->byteSize(); +} + +inline bool LStackArea::ResultIterator::done() const { + return idx_ == alloc_.ins()->toWasmStackResultArea()->mir()->resultCount(); +} +inline void LStackArea::ResultIterator::next() { + MOZ_ASSERT(!done()); + idx_++; +} +inline LAllocation LStackArea::ResultIterator::alloc() const { + MOZ_ASSERT(!done()); + MWasmStackResultArea* area = alloc_.ins()->toWasmStackResultArea()->mir(); + return LStackSlot(area->base() - area->result(idx_).offset()); +} +inline bool LStackArea::ResultIterator::isGcPointer() const { + MOZ_ASSERT(!done()); + MWasmStackResultArea* area = alloc_.ins()->toWasmStackResultArea()->mir(); + MIRType type = area->result(idx_).type(); +#ifndef JS_PUNBOX64 + // LDefinition::TypeFrom isn't defined for MIRType::Int64 values on + // this platform, so here we have a special case. + if (type == MIRType::Int64) { + return false; + } +#endif + return LDefinition::TypeFrom(type) == LDefinition::OBJECT; +} + +class LWasmStackResult : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(WasmStackResult); + + LWasmStackResult() : LInstructionHelper(classOpcode) {} + + MWasmStackResult* mir() const { return mir_->toWasmStackResult(); } + LStackSlot result(uint32_t base) const { + return LStackSlot(base - mir()->result().offset()); + } +}; + +class LWasmStackResult64 : public LInstructionHelper<INT64_PIECES, 1, 0> { + public: + LIR_HEADER(WasmStackResult64); + + LWasmStackResult64() : LInstructionHelper(classOpcode) {} + + MWasmStackResult* mir() const { return mir_->toWasmStackResult(); } + LStackSlot result(uint32_t base, LDefinition* def) { + uint32_t offset = base - mir()->result().offset(); +#if defined(JS_NUNBOX32) + if (def == getDef(INT64LOW_INDEX)) { + offset -= INT64LOW_OFFSET; + } else { + MOZ_ASSERT(def == getDef(INT64HIGH_INDEX)); + offset -= INT64HIGH_OFFSET; + } +#else + MOZ_ASSERT(def == getDef(0)); +#endif + return LStackSlot(offset); + } +}; + +inline LStackSlot LStackArea::resultAlloc(LInstruction* lir, + LDefinition* def) const { + if (lir->isWasmStackResult64()) { + return lir->toWasmStackResult64()->result(base(), def); + } + MOZ_ASSERT(def == lir->getDef(0)); + return lir->toWasmStackResult()->result(base()); +} + +inline bool LNode::isCallPreserved(AnyRegister reg) const { + return isWasmCall() && LWasmCall::isCallPreserved(reg); +} + +class LAssertRangeI : public LInstructionHelper<0, 1, 0> { + public: + LIR_HEADER(AssertRangeI) + + explicit LAssertRangeI(const LAllocation& input) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + } + + const LAllocation* input() { return getOperand(0); } + + MAssertRange* mir() { return mir_->toAssertRange(); } + const Range* range() { return mir()->assertedRange(); } +}; + +class LAssertRangeD : public LInstructionHelper<0, 1, 1> { + public: + LIR_HEADER(AssertRangeD) + + LAssertRangeD(const LAllocation& input, const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + setTemp(0, temp); + } + + const LAllocation* input() { return getOperand(0); } + + const LDefinition* temp() { return getTemp(0); } + + MAssertRange* mir() { return mir_->toAssertRange(); } + const Range* range() { return mir()->assertedRange(); } +}; + +class LAssertRangeF : public LInstructionHelper<0, 1, 2> { + public: + LIR_HEADER(AssertRangeF) + LAssertRangeF(const LAllocation& input, const LDefinition& temp, + const LDefinition& temp2) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + setTemp(0, temp); + setTemp(1, temp2); + } + + const LAllocation* input() { return getOperand(0); } + const LDefinition* temp() { return getTemp(0); } + const LDefinition* temp2() { return getTemp(1); } + + MAssertRange* mir() { return mir_->toAssertRange(); } + const Range* range() { return mir()->assertedRange(); } +}; + +class LAssertRangeV : public LInstructionHelper<0, BOX_PIECES, 3> { + public: + LIR_HEADER(AssertRangeV) + + LAssertRangeV(const LBoxAllocation& input, const LDefinition& temp, + const LDefinition& floatTemp1, const LDefinition& floatTemp2) + : LInstructionHelper(classOpcode) { + setBoxOperand(Input, input); + setTemp(0, temp); + setTemp(1, floatTemp1); + setTemp(2, floatTemp2); + } + + static const size_t Input = 0; + + const LDefinition* temp() { return getTemp(0); } + const LDefinition* floatTemp1() { return getTemp(1); } + const LDefinition* floatTemp2() { return getTemp(2); } + + MAssertRange* mir() { return mir_->toAssertRange(); } + const Range* range() { return mir()->assertedRange(); } +}; + +class LMemoryBarrier : public LInstructionHelper<0, 0, 0> { + private: + const MemoryBarrierBits type_; + + public: + LIR_HEADER(MemoryBarrier) + + // The parameter 'type' is a bitwise 'or' of the barrier types needed, + // see AtomicOp.h. + explicit LMemoryBarrier(MemoryBarrierBits type) + : LInstructionHelper(classOpcode), type_(type) { + MOZ_ASSERT((type_ & ~MembarAllbits) == MembarNobits); + } + + MemoryBarrierBits type() const { return type_; } +}; + +// Math.random(). +class LRandom : public LInstructionHelper<1, 0, 1 + 2 * INT64_PIECES> { + public: + LIR_HEADER(Random) + LRandom(const LDefinition& temp0, const LInt64Definition& temp1, + const LInt64Definition& temp2) + : LInstructionHelper(classOpcode) { + setTemp(0, temp0); + setInt64Temp(1, temp1); + setInt64Temp(1 + INT64_PIECES, temp2); + } + const LDefinition* temp0() { return getTemp(0); } + LInt64Definition temp1() { return getInt64Temp(1); } + LInt64Definition temp2() { return getInt64Temp(1 + INT64_PIECES); } + + MRandom* mir() const { return mir_->toRandom(); } +}; + +class LBigIntAsIntN64 : public LInstructionHelper<1, 1, 1 + INT64_PIECES> { + public: + LIR_HEADER(BigIntAsIntN64) + + LBigIntAsIntN64(const LAllocation& input, const LDefinition& temp, + const LInt64Definition& temp64) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + setTemp(0, temp); + setInt64Temp(1, temp64); + } + + const LAllocation* input() { return getOperand(0); } + const LDefinition* temp() { return getTemp(0); } + LInt64Definition temp64() { return getInt64Temp(1); } +}; + +class LBigIntAsIntN32 : public LInstructionHelper<1, 1, 1 + INT64_PIECES> { + public: + LIR_HEADER(BigIntAsIntN32) + + LBigIntAsIntN32(const LAllocation& input, const LDefinition& temp, + const LInt64Definition& temp64) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + setTemp(0, temp); + setInt64Temp(1, temp64); + } + + const LAllocation* input() { return getOperand(0); } + const LDefinition* temp() { return getTemp(0); } + LInt64Definition temp64() { return getInt64Temp(1); } +}; + +class LBigIntAsUintN64 : public LInstructionHelper<1, 1, 1 + INT64_PIECES> { + public: + LIR_HEADER(BigIntAsUintN64) + + LBigIntAsUintN64(const LAllocation& input, const LDefinition& temp, + const LInt64Definition& temp64) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + setTemp(0, temp); + setInt64Temp(1, temp64); + } + + const LAllocation* input() { return getOperand(0); } + const LDefinition* temp() { return getTemp(0); } + LInt64Definition temp64() { return getInt64Temp(1); } +}; + +class LBigIntAsUintN32 : public LInstructionHelper<1, 1, 1 + INT64_PIECES> { + public: + LIR_HEADER(BigIntAsUintN32) + + LBigIntAsUintN32(const LAllocation& input, const LDefinition& temp, + const LInt64Definition& temp64) + : LInstructionHelper(classOpcode) { + setOperand(0, input); + setTemp(0, temp); + setInt64Temp(1, temp64); + } + + const LAllocation* input() { return getOperand(0); } + const LDefinition* temp() { return getTemp(0); } + LInt64Definition temp64() { return getInt64Temp(1); } +}; + +template <size_t NumDefs> +class LIonToWasmCallBase : public LVariadicInstruction<NumDefs, 1> { + using Base = LVariadicInstruction<NumDefs, 1>; + + public: + explicit LIonToWasmCallBase(LNode::Opcode classOpcode, uint32_t numOperands, + const LDefinition& temp) + : Base(classOpcode, numOperands) { + this->setIsCall(); + this->setTemp(0, temp); + } + MIonToWasmCall* mir() const { return this->mir_->toIonToWasmCall(); } + const LDefinition* temp() { return this->getTemp(0); } +}; + +class LIonToWasmCall : public LIonToWasmCallBase<1> { + public: + LIR_HEADER(IonToWasmCall); + LIonToWasmCall(uint32_t numOperands, const LDefinition& temp) + : LIonToWasmCallBase<1>(classOpcode, numOperands, temp) {} +}; + +class LIonToWasmCallV : public LIonToWasmCallBase<BOX_PIECES> { + public: + LIR_HEADER(IonToWasmCallV); + LIonToWasmCallV(uint32_t numOperands, const LDefinition& temp) + : LIonToWasmCallBase<BOX_PIECES>(classOpcode, numOperands, temp) {} +}; + +class LIonToWasmCallI64 : public LIonToWasmCallBase<INT64_PIECES> { + public: + LIR_HEADER(IonToWasmCallI64); + LIonToWasmCallI64(uint32_t numOperands, const LDefinition& temp) + : LIonToWasmCallBase<INT64_PIECES>(classOpcode, numOperands, temp) {} +}; + +class LWasmGcObjectIsSubtypeOfAbstractAndBranch + : public LControlInstructionHelper<2, 2, 2> { + wasm::RefType sourceType_; + wasm::RefType destType_; + + public: + LIR_HEADER(WasmGcObjectIsSubtypeOfAbstractAndBranch) + + static constexpr uint32_t Object = 0; + + LWasmGcObjectIsSubtypeOfAbstractAndBranch(MBasicBlock* ifTrue, + MBasicBlock* ifFalse, + wasm::RefType sourceType, + wasm::RefType destType, + const LAllocation& object, + const LDefinition& temp0) + : LControlInstructionHelper(classOpcode), + sourceType_(sourceType), + destType_(destType) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setOperand(Object, object); + setTemp(0, temp0); + } + + wasm::RefType sourceType() const { return sourceType_; } + wasm::RefType destType() const { return destType_; } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + + const LAllocation* object() { return getOperand(Object); } + const LDefinition* temp0() { return getTemp(0); } +}; + +class LWasmGcObjectIsSubtypeOfConcreteAndBranch + : public LControlInstructionHelper<2, 2, 2> { + wasm::RefType sourceType_; + wasm::RefType destType_; + + public: + LIR_HEADER(WasmGcObjectIsSubtypeOfConcreteAndBranch) + + static constexpr uint32_t Object = 0; + static constexpr uint32_t SuperSuperTypeVector = 1; + + LWasmGcObjectIsSubtypeOfConcreteAndBranch( + MBasicBlock* ifTrue, MBasicBlock* ifFalse, wasm::RefType sourceType, + wasm::RefType destType, const LAllocation& object, + const LAllocation& superSuperTypeVector, const LDefinition& temp0, + const LDefinition& temp1) + : LControlInstructionHelper(classOpcode), + sourceType_(sourceType), + destType_(destType) { + setSuccessor(0, ifTrue); + setSuccessor(1, ifFalse); + setOperand(Object, object); + setOperand(SuperSuperTypeVector, superSuperTypeVector); + setTemp(0, temp0); + setTemp(1, temp1); + } + + wasm::RefType sourceType() const { return sourceType_; } + wasm::RefType destType() const { return destType_; } + + MBasicBlock* ifTrue() const { return getSuccessor(0); } + MBasicBlock* ifFalse() const { return getSuccessor(1); } + + const LAllocation* object() { return getOperand(Object); } + const LAllocation* superSuperTypeVector() { + return getOperand(SuperSuperTypeVector); + } + const LDefinition* temp0() { return getTemp(0); } + const LDefinition* temp1() { return getTemp(1); } +}; + +// Wasm SIMD. + +// (v128, v128, v128) -> v128 effect-free operation. +// temp is FPR. +class LWasmTernarySimd128 : public LInstructionHelper<1, 3, 1> { + wasm::SimdOp op_; + + public: + LIR_HEADER(WasmTernarySimd128) + + static constexpr uint32_t V0 = 0; + static constexpr uint32_t V1 = 1; + static constexpr uint32_t V2 = 2; + + LWasmTernarySimd128(wasm::SimdOp op, const LAllocation& v0, + const LAllocation& v1, const LAllocation& v2) + : LInstructionHelper(classOpcode), op_(op) { + setOperand(V0, v0); + setOperand(V1, v1); + setOperand(V2, v2); + } + + LWasmTernarySimd128(wasm::SimdOp op, const LAllocation& v0, + const LAllocation& v1, const LAllocation& v2, + const LDefinition& temp) + : LInstructionHelper(classOpcode), op_(op) { + setOperand(V0, v0); + setOperand(V1, v1); + setOperand(V2, v2); + setTemp(0, temp); + } + + const LAllocation* v0() { return getOperand(V0); } + const LAllocation* v1() { return getOperand(V1); } + const LAllocation* v2() { return getOperand(V2); } + const LDefinition* temp() { return getTemp(0); } + + wasm::SimdOp simdOp() const { return op_; } +}; + +// (v128, v128) -> v128 effect-free operations +// lhs and dest are the same. +// temps (if in use) are FPR. +// The op may differ from the MIR node's op. +class LWasmBinarySimd128 : public LInstructionHelper<1, 2, 2> { + wasm::SimdOp op_; + + public: + LIR_HEADER(WasmBinarySimd128) + + static constexpr uint32_t Lhs = 0; + static constexpr uint32_t LhsDest = 0; + static constexpr uint32_t Rhs = 1; + + LWasmBinarySimd128(wasm::SimdOp op, const LAllocation& lhs, + const LAllocation& rhs, const LDefinition& temp0, + const LDefinition& temp1) + : LInstructionHelper(classOpcode), op_(op) { + setOperand(Lhs, lhs); + setOperand(Rhs, rhs); + setTemp(0, temp0); + setTemp(1, temp1); + } + + const LAllocation* lhs() { return getOperand(Lhs); } + const LAllocation* lhsDest() { return getOperand(LhsDest); } + const LAllocation* rhs() { return getOperand(Rhs); } + wasm::SimdOp simdOp() const { return op_; } + + static bool SpecializeForConstantRhs(wasm::SimdOp op); +}; + +class LWasmBinarySimd128WithConstant : public LInstructionHelper<1, 1, 1> { + SimdConstant rhs_; + + public: + LIR_HEADER(WasmBinarySimd128WithConstant) + + static constexpr uint32_t Lhs = 0; + static constexpr uint32_t LhsDest = 0; + + LWasmBinarySimd128WithConstant(const LAllocation& lhs, + const SimdConstant& rhs, + const LDefinition& temp) + : LInstructionHelper(classOpcode), rhs_(rhs) { + setOperand(Lhs, lhs); + setTemp(0, temp); + } + + const LAllocation* lhs() { return getOperand(Lhs); } + const LAllocation* lhsDest() { return getOperand(LhsDest); } + const SimdConstant& rhs() { return rhs_; } + wasm::SimdOp simdOp() const { + return mir_->toWasmBinarySimd128WithConstant()->simdOp(); + } +}; + +// (v128, i32) -> v128 effect-free variable-width shift operations +// lhs and dest are the same. +// temp is an FPR (if in use). +class LWasmVariableShiftSimd128 : public LInstructionHelper<1, 2, 1> { + public: + LIR_HEADER(WasmVariableShiftSimd128) + + static constexpr uint32_t Lhs = 0; + static constexpr uint32_t LhsDest = 0; + static constexpr uint32_t Rhs = 1; + + LWasmVariableShiftSimd128(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(Lhs, lhs); + setOperand(Rhs, rhs); + setTemp(0, temp); + } + + const LAllocation* lhs() { return getOperand(Lhs); } + const LAllocation* lhsDest() { return getOperand(LhsDest); } + const LAllocation* rhs() { return getOperand(Rhs); } + wasm::SimdOp simdOp() const { return mir_->toWasmShiftSimd128()->simdOp(); } +}; + +// (v128, i32) -> v128 effect-free constant-width shift operations +class LWasmConstantShiftSimd128 : public LInstructionHelper<1, 1, 0> { + int32_t shift_; + + public: + LIR_HEADER(WasmConstantShiftSimd128) + + static constexpr uint32_t Src = 0; + + LWasmConstantShiftSimd128(const LAllocation& src, int32_t shift) + : LInstructionHelper(classOpcode), shift_(shift) { + setOperand(Src, src); + } + + const LAllocation* src() { return getOperand(Src); } + int32_t shift() { return shift_; } + wasm::SimdOp simdOp() const { return mir_->toWasmShiftSimd128()->simdOp(); } +}; + +// (v128) -> v128 sign replication operation. +class LWasmSignReplicationSimd128 : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(WasmSignReplicationSimd128) + + static constexpr uint32_t Src = 0; + + explicit LWasmSignReplicationSimd128(const LAllocation& src) + : LInstructionHelper(classOpcode) { + setOperand(Src, src); + } + + const LAllocation* src() { return getOperand(Src); } + wasm::SimdOp simdOp() const { return mir_->toWasmShiftSimd128()->simdOp(); } +}; + +// (v128, v128, imm_simd) -> v128 effect-free operation. +// temp is FPR (and always in use). +class LWasmShuffleSimd128 : public LInstructionHelper<1, 2, 1> { + private: + SimdShuffleOp op_; + SimdConstant control_; + + public: + LIR_HEADER(WasmShuffleSimd128) + + static constexpr uint32_t Lhs = 0; + static constexpr uint32_t LhsDest = 0; + static constexpr uint32_t Rhs = 1; + + LWasmShuffleSimd128(const LAllocation& lhs, const LAllocation& rhs, + const LDefinition& temp, SimdShuffleOp op, + SimdConstant control) + : LInstructionHelper(classOpcode), op_(op), control_(control) { + setOperand(Lhs, lhs); + setOperand(Rhs, rhs); + setTemp(0, temp); + } + + const LAllocation* lhs() { return getOperand(Lhs); } + const LAllocation* lhsDest() { return getOperand(LhsDest); } + const LAllocation* rhs() { return getOperand(Rhs); } + const LDefinition* temp() { return getTemp(0); } + SimdShuffleOp op() { return op_; } + SimdConstant control() { return control_; } +}; + +// (v128, imm_simd) -> v128 effect-free operation. +class LWasmPermuteSimd128 : public LInstructionHelper<1, 1, 0> { + private: + SimdPermuteOp op_; + SimdConstant control_; + + public: + LIR_HEADER(WasmPermuteSimd128) + + static constexpr uint32_t Src = 0; + + LWasmPermuteSimd128(const LAllocation& src, SimdPermuteOp op, + SimdConstant control) + : LInstructionHelper(classOpcode), op_(op), control_(control) { + setOperand(Src, src); + } + + const LAllocation* src() { return getOperand(Src); } + SimdPermuteOp op() { return op_; } + SimdConstant control() { return control_; } +}; + +class LWasmReplaceLaneSimd128 : public LInstructionHelper<1, 2, 0> { + public: + LIR_HEADER(WasmReplaceLaneSimd128) + + static constexpr uint32_t Lhs = 0; + static constexpr uint32_t LhsDest = 0; + static constexpr uint32_t Rhs = 1; + + LWasmReplaceLaneSimd128(const LAllocation& lhs, const LAllocation& rhs) + : LInstructionHelper(classOpcode) { + setOperand(Lhs, lhs); + setOperand(Rhs, rhs); + } + + const LAllocation* lhs() { return getOperand(Lhs); } + const LAllocation* lhsDest() { return getOperand(LhsDest); } + const LAllocation* rhs() { return getOperand(Rhs); } + uint32_t laneIndex() const { + return mir_->toWasmReplaceLaneSimd128()->laneIndex(); + } + wasm::SimdOp simdOp() const { + return mir_->toWasmReplaceLaneSimd128()->simdOp(); + } +}; + +class LWasmReplaceInt64LaneSimd128 + : public LInstructionHelper<1, INT64_PIECES + 1, 0> { + public: + LIR_HEADER(WasmReplaceInt64LaneSimd128) + + static constexpr uint32_t Lhs = 0; + static constexpr uint32_t LhsDest = 0; + static constexpr uint32_t Rhs = 1; + + LWasmReplaceInt64LaneSimd128(const LAllocation& lhs, + const LInt64Allocation& rhs) + : LInstructionHelper(classOpcode) { + setOperand(Lhs, lhs); + setInt64Operand(Rhs, rhs); + } + + const LAllocation* lhs() { return getOperand(Lhs); } + const LAllocation* lhsDest() { return getOperand(LhsDest); } + const LInt64Allocation rhs() { return getInt64Operand(Rhs); } + const LDefinition* output() { return this->getDef(0); } + uint32_t laneIndex() const { + return mir_->toWasmReplaceLaneSimd128()->laneIndex(); + } + wasm::SimdOp simdOp() const { + return mir_->toWasmReplaceLaneSimd128()->simdOp(); + } +}; + +// (scalar) -> v128 effect-free operations, scalar != int64 +class LWasmScalarToSimd128 : public LInstructionHelper<1, 1, 0> { + public: + LIR_HEADER(WasmScalarToSimd128) + + static constexpr uint32_t Src = 0; + + explicit LWasmScalarToSimd128(const LAllocation& src) + : LInstructionHelper(classOpcode) { + setOperand(Src, src); + } + + const LAllocation* src() { return getOperand(Src); } + wasm::SimdOp simdOp() const { + return mir_->toWasmScalarToSimd128()->simdOp(); + } +}; + +// (int64) -> v128 effect-free operations +class LWasmInt64ToSimd128 : public LInstructionHelper<1, INT64_PIECES, 0> { + public: + LIR_HEADER(WasmInt64ToSimd128) + + static constexpr uint32_t Src = 0; + + explicit LWasmInt64ToSimd128(const LInt64Allocation& src) + : LInstructionHelper(classOpcode) { + setInt64Operand(Src, src); + } + + const LInt64Allocation src() { return getInt64Operand(Src); } + wasm::SimdOp simdOp() const { + return mir_->toWasmScalarToSimd128()->simdOp(); + } +}; + +// (v128) -> v128 effect-free operations +// temp is FPR (if in use). +class LWasmUnarySimd128 : public LInstructionHelper<1, 1, 1> { + public: + LIR_HEADER(WasmUnarySimd128) + + static constexpr uint32_t Src = 0; + + LWasmUnarySimd128(const LAllocation& src, const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(Src, src); + setTemp(0, temp); + } + + const LAllocation* src() { return getOperand(Src); } + const LDefinition* temp() { return getTemp(0); } + wasm::SimdOp simdOp() const { return mir_->toWasmUnarySimd128()->simdOp(); } +}; + +// (v128, imm) -> scalar effect-free operations. +// temp is FPR (if in use). +class LWasmReduceSimd128 : public LInstructionHelper<1, 1, 1> { + public: + LIR_HEADER(WasmReduceSimd128) + + static constexpr uint32_t Src = 0; + + explicit LWasmReduceSimd128(const LAllocation& src, const LDefinition& temp) + : LInstructionHelper(classOpcode) { + setOperand(Src, src); + setTemp(0, temp); + } + + const LAllocation* src() { return getOperand(Src); } + uint32_t imm() const { return mir_->toWasmReduceSimd128()->imm(); } + wasm::SimdOp simdOp() const { return mir_->toWasmReduceSimd128()->simdOp(); } +}; + +// (v128, onTrue, onFalse) test-and-branch operations. +class LWasmReduceAndBranchSimd128 : public LControlInstructionHelper<2, 1, 0> { + wasm::SimdOp op_; + + public: + LIR_HEADER(WasmReduceAndBranchSimd128) + + static constexpr uint32_t Src = 0; + static constexpr uint32_t IfTrue = 0; + static constexpr uint32_t IfFalse = 1; + + LWasmReduceAndBranchSimd128(const LAllocation& src, wasm::SimdOp op, + MBasicBlock* ifTrue, MBasicBlock* ifFalse) + : LControlInstructionHelper(classOpcode), op_(op) { + setOperand(Src, src); + setSuccessor(IfTrue, ifTrue); + setSuccessor(IfFalse, ifFalse); + } + + const LAllocation* src() { return getOperand(Src); } + wasm::SimdOp simdOp() const { return op_; } + MBasicBlock* ifTrue() const { return getSuccessor(IfTrue); } + MBasicBlock* ifFalse() const { return getSuccessor(IfFalse); } +}; + +// (v128, imm) -> i64 effect-free operations +class LWasmReduceSimd128ToInt64 + : public LInstructionHelper<INT64_PIECES, 1, 0> { + public: + LIR_HEADER(WasmReduceSimd128ToInt64) + + static constexpr uint32_t Src = 0; + + explicit LWasmReduceSimd128ToInt64(const LAllocation& src) + : LInstructionHelper(classOpcode) { + setOperand(Src, src); + } + + const LAllocation* src() { return getOperand(Src); } + uint32_t imm() const { return mir_->toWasmReduceSimd128()->imm(); } + wasm::SimdOp simdOp() const { return mir_->toWasmReduceSimd128()->simdOp(); } +}; + +class LWasmLoadLaneSimd128 : public LInstructionHelper<1, 3, 1> { + public: + LIR_HEADER(WasmLoadLaneSimd128); + + static constexpr uint32_t Src = 2; + + explicit LWasmLoadLaneSimd128(const LAllocation& ptr, const LAllocation& src, + const LDefinition& temp, + const LAllocation& memoryBase = LAllocation()) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, memoryBase); + setOperand(Src, src); + setTemp(0, temp); + } + + const LAllocation* ptr() { return getOperand(0); } + const LAllocation* memoryBase() { return getOperand(1); } + const LAllocation* src() { return getOperand(Src); } + const LDefinition* temp() { return getTemp(0); } + MWasmLoadLaneSimd128* mir() const { return mir_->toWasmLoadLaneSimd128(); } + uint32_t laneSize() const { + return mir_->toWasmLoadLaneSimd128()->laneSize(); + } + uint32_t laneIndex() const { + return mir_->toWasmLoadLaneSimd128()->laneIndex(); + } +}; + +class LWasmStoreLaneSimd128 : public LInstructionHelper<1, 3, 1> { + public: + LIR_HEADER(WasmStoreLaneSimd128); + + static constexpr uint32_t Src = 2; + + explicit LWasmStoreLaneSimd128(const LAllocation& ptr, const LAllocation& src, + const LDefinition& temp, + const LAllocation& memoryBase = LAllocation()) + : LInstructionHelper(classOpcode) { + setOperand(0, ptr); + setOperand(1, memoryBase); + setOperand(Src, src); + setTemp(0, temp); + } + + const LAllocation* ptr() { return getOperand(0); } + const LAllocation* memoryBase() { return getOperand(1); } + const LAllocation* src() { return getOperand(Src); } + const LDefinition* temp() { return getTemp(0); } + MWasmStoreLaneSimd128* mir() const { return mir_->toWasmStoreLaneSimd128(); } + uint32_t laneSize() const { + return mir_->toWasmStoreLaneSimd128()->laneSize(); + } + uint32_t laneIndex() const { + return mir_->toWasmStoreLaneSimd128()->laneIndex(); + } +}; + +// End Wasm SIMD + +// End Wasm Exception Handling + +} // namespace jit +} // namespace js + +#endif /* jit_shared_LIR_shared_h */ diff --git a/js/src/jit/shared/Lowering-shared-inl.h b/js/src/jit/shared/Lowering-shared-inl.h new file mode 100644 index 0000000000..89aafa22e4 --- /dev/null +++ b/js/src/jit/shared/Lowering-shared-inl.h @@ -0,0 +1,894 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_Lowering_shared_inl_h +#define jit_shared_Lowering_shared_inl_h + +#include "jit/shared/Lowering-shared.h" + +#include "jit/MIR.h" +#include "jit/MIRGenerator.h" + +namespace js { +namespace jit { + +void LIRGeneratorShared::emitAtUses(MInstruction* mir) { + MOZ_ASSERT(mir->canEmitAtUses()); + mir->setEmittedAtUses(); + mir->setVirtualRegister(0); +} + +LUse LIRGeneratorShared::use(MDefinition* mir, LUse policy) { + // It is illegal to call use() on an instruction with two defs. +#if BOX_PIECES > 1 + MOZ_ASSERT(mir->type() != MIRType::Value); +#endif +#if INT64_PIECES > 1 + MOZ_ASSERT(mir->type() != MIRType::Int64); +#endif + ensureDefined(mir); + policy.setVirtualRegister(mir->virtualRegister()); + return policy; +} + +template <size_t X> +void LIRGeneratorShared::define( + details::LInstructionFixedDefsTempsHelper<1, X>* lir, MDefinition* mir, + LDefinition::Policy policy) { + LDefinition::Type type = LDefinition::TypeFrom(mir->type()); + define(lir, mir, LDefinition(type, policy)); +} + +template <size_t X> +void LIRGeneratorShared::define( + details::LInstructionFixedDefsTempsHelper<1, X>* lir, MDefinition* mir, + const LDefinition& def) { + // Call instructions should use defineReturn. + MOZ_ASSERT(!lir->isCall()); + + uint32_t vreg = getVirtualRegister(); + + // Assign the definition and a virtual register. Then, propagate this + // virtual register to the MIR, so we can map MIR to LIR during lowering. + lir->setDef(0, def); + lir->getDef(0)->setVirtualRegister(vreg); + lir->setMir(mir); + mir->setVirtualRegister(vreg); + add(lir); +} + +template <size_t X, size_t Y> +void LIRGeneratorShared::defineFixed(LInstructionHelper<1, X, Y>* lir, + MDefinition* mir, + const LAllocation& output) { + LDefinition::Type type = LDefinition::TypeFrom(mir->type()); + + LDefinition def(type, LDefinition::FIXED); + def.setOutput(output); + + define(lir, mir, def); +} + +template <size_t Ops, size_t Temps> +void LIRGeneratorShared::defineInt64Fixed( + LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir, + const LInt64Allocation& output) { + uint32_t vreg = getVirtualRegister(); + +#if JS_BITS_PER_WORD == 64 + LDefinition def(LDefinition::GENERAL, LDefinition::FIXED); + def.setOutput(output.value()); + lir->setDef(0, def); + lir->getDef(0)->setVirtualRegister(vreg); +#else + LDefinition def0(LDefinition::GENERAL, LDefinition::FIXED); + def0.setOutput(output.low()); + lir->setDef(0, def0); + lir->getDef(0)->setVirtualRegister(vreg); + + getVirtualRegister(); + LDefinition def1(LDefinition::GENERAL, LDefinition::FIXED); + def1.setOutput(output.high()); + lir->setDef(1, def1); + lir->getDef(1)->setVirtualRegister(vreg + 1); +#endif + + lir->setMir(mir); + mir->setVirtualRegister(vreg); + add(lir); +} + +template <size_t Ops, size_t Temps> +void LIRGeneratorShared::defineReuseInput( + LInstructionHelper<1, Ops, Temps>* lir, MDefinition* mir, + uint32_t operand) { + // Note: Any other operand that is not the same as this operand should be + // marked as not being "atStart". The regalloc cannot handle those and can + // overwrite the inputs! + + // The input should be used at the start of the instruction, to avoid moves. + MOZ_ASSERT(lir->getOperand(operand)->toUse()->usedAtStart()); + + LDefinition::Type type = LDefinition::TypeFrom(mir->type()); + + LDefinition def(type, LDefinition::MUST_REUSE_INPUT); + def.setReusedInput(operand); + + define(lir, mir, def); +} + +template <size_t Ops, size_t Temps> +void LIRGeneratorShared::defineInt64ReuseInput( + LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir, + uint32_t operand) { + // Note: Any other operand that is not the same as this operand should be + // marked as not being "atStart". The regalloc cannot handle those and can + // overwrite the inputs! + + // The input should be used at the start of the instruction, to avoid moves. + MOZ_ASSERT(lir->getOperand(operand)->toUse()->usedAtStart()); +#if JS_BITS_PER_WORD == 32 + MOZ_ASSERT(lir->getOperand(operand + 1)->toUse()->usedAtStart()); +#endif + MOZ_ASSERT(!lir->isCall()); + + uint32_t vreg = getVirtualRegister(); + + LDefinition def1(LDefinition::GENERAL, LDefinition::MUST_REUSE_INPUT); + def1.setReusedInput(operand); + lir->setDef(0, def1); + lir->getDef(0)->setVirtualRegister(vreg); + +#if JS_BITS_PER_WORD == 32 + getVirtualRegister(); + LDefinition def2(LDefinition::GENERAL, LDefinition::MUST_REUSE_INPUT); + def2.setReusedInput(operand + 1); + lir->setDef(1, def2); + lir->getDef(1)->setVirtualRegister(vreg + 1); +#endif + + lir->setMir(mir); + mir->setVirtualRegister(vreg); + add(lir); +} + +template <size_t Ops, size_t Temps> +void LIRGeneratorShared::defineBoxReuseInput( + LInstructionHelper<BOX_PIECES, Ops, Temps>* lir, MDefinition* mir, + uint32_t operand) { + // The input should be used at the start of the instruction, to avoid moves. + MOZ_ASSERT(lir->getOperand(operand)->toUse()->usedAtStart()); +#ifdef JS_NUNBOX32 + MOZ_ASSERT(lir->getOperand(operand + 1)->toUse()->usedAtStart()); +#endif + MOZ_ASSERT(!lir->isCall()); + MOZ_ASSERT(mir->type() == MIRType::Value); + + uint32_t vreg = getVirtualRegister(); + +#ifdef JS_NUNBOX32 + static_assert(VREG_TYPE_OFFSET == 0, + "Code below assumes VREG_TYPE_OFFSET == 0"); + static_assert(VREG_DATA_OFFSET == 1, + "Code below assumes VREG_DATA_OFFSET == 1"); + + LDefinition def1(LDefinition::TYPE, LDefinition::MUST_REUSE_INPUT); + def1.setReusedInput(operand); + def1.setVirtualRegister(vreg); + lir->setDef(0, def1); + + getVirtualRegister(); + LDefinition def2(LDefinition::PAYLOAD, LDefinition::MUST_REUSE_INPUT); + def2.setReusedInput(operand + 1); + def2.setVirtualRegister(vreg + 1); + lir->setDef(1, def2); +#else + LDefinition def(LDefinition::BOX, LDefinition::MUST_REUSE_INPUT); + def.setReusedInput(operand); + def.setVirtualRegister(vreg); + lir->setDef(0, def); +#endif + + lir->setMir(mir); + mir->setVirtualRegister(vreg); + add(lir); +} + +template <size_t Temps> +void LIRGeneratorShared::defineBox( + details::LInstructionFixedDefsTempsHelper<BOX_PIECES, Temps>* lir, + MDefinition* mir, LDefinition::Policy policy) { + // Call instructions should use defineReturn. + MOZ_ASSERT(!lir->isCall()); + MOZ_ASSERT(mir->type() == MIRType::Value); + + uint32_t vreg = getVirtualRegister(); + +#if defined(JS_NUNBOX32) + lir->setDef(0, + LDefinition(vreg + VREG_TYPE_OFFSET, LDefinition::TYPE, policy)); + lir->setDef( + 1, LDefinition(vreg + VREG_DATA_OFFSET, LDefinition::PAYLOAD, policy)); + getVirtualRegister(); +#elif defined(JS_PUNBOX64) + lir->setDef(0, LDefinition(vreg, LDefinition::BOX, policy)); +#endif + lir->setMir(mir); + + mir->setVirtualRegister(vreg); + add(lir); +} + +template <size_t Ops, size_t Temps> +void LIRGeneratorShared::defineInt64( + LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir, + LDefinition::Policy policy) { + // Call instructions should use defineReturn. + MOZ_ASSERT(!lir->isCall()); + +#ifdef JS_64BIT + MOZ_ASSERT(mir->type() == MIRType::Int64 || mir->type() == MIRType::IntPtr); +#else + MOZ_ASSERT(mir->type() == MIRType::Int64); +#endif + + uint32_t vreg = getVirtualRegister(); + +#if JS_BITS_PER_WORD == 32 + lir->setDef(0, + LDefinition(vreg + INT64LOW_INDEX, LDefinition::GENERAL, policy)); + lir->setDef( + 1, LDefinition(vreg + INT64HIGH_INDEX, LDefinition::GENERAL, policy)); + getVirtualRegister(); +#else + lir->setDef(0, LDefinition(vreg, LDefinition::GENERAL, policy)); +#endif + lir->setMir(mir); + + mir->setVirtualRegister(vreg); + add(lir); +} + +void LIRGeneratorShared::defineReturn(LInstruction* lir, MDefinition* mir) { + lir->setMir(mir); + + MOZ_ASSERT(lir->isCall()); + + uint32_t vreg = getVirtualRegister(); + + switch (mir->type()) { + case MIRType::Value: +#if defined(JS_NUNBOX32) + lir->setDef(TYPE_INDEX, + LDefinition(vreg + VREG_TYPE_OFFSET, LDefinition::TYPE, + LGeneralReg(JSReturnReg_Type))); + lir->setDef(PAYLOAD_INDEX, + LDefinition(vreg + VREG_DATA_OFFSET, LDefinition::PAYLOAD, + LGeneralReg(JSReturnReg_Data))); + getVirtualRegister(); +#elif defined(JS_PUNBOX64) + lir->setDef( + 0, LDefinition(vreg, LDefinition::BOX, LGeneralReg(JSReturnReg))); +#endif + break; + case MIRType::Int64: +#if defined(JS_NUNBOX32) + lir->setDef(INT64LOW_INDEX, + LDefinition(vreg + INT64LOW_INDEX, LDefinition::GENERAL, + LGeneralReg(ReturnReg64.low))); + lir->setDef(INT64HIGH_INDEX, + LDefinition(vreg + INT64HIGH_INDEX, LDefinition::GENERAL, + LGeneralReg(ReturnReg64.high))); + getVirtualRegister(); +#elif defined(JS_PUNBOX64) + lir->setDef( + 0, LDefinition(vreg, LDefinition::GENERAL, LGeneralReg(ReturnReg))); +#endif + break; + case MIRType::Float32: + lir->setDef(0, LDefinition(vreg, LDefinition::FLOAT32, + LFloatReg(ReturnFloat32Reg))); + break; + case MIRType::Double: + lir->setDef(0, LDefinition(vreg, LDefinition::DOUBLE, + LFloatReg(ReturnDoubleReg))); + break; + case MIRType::Simd128: +#ifdef ENABLE_WASM_SIMD + lir->setDef(0, LDefinition(vreg, LDefinition::SIMD128, + LFloatReg(ReturnSimd128Reg))); + break; +#else + MOZ_CRASH("No SIMD support"); +#endif + default: + LDefinition::Type type = LDefinition::TypeFrom(mir->type()); + switch (type) { + case LDefinition::GENERAL: + case LDefinition::INT32: + case LDefinition::OBJECT: + case LDefinition::SLOTS: + case LDefinition::STACKRESULTS: + lir->setDef(0, LDefinition(vreg, type, LGeneralReg(ReturnReg))); + break; + case LDefinition::DOUBLE: + case LDefinition::FLOAT32: + case LDefinition::SIMD128: + MOZ_CRASH("Float cases must have been handled earlier"); + default: + MOZ_CRASH("Unexpected type"); + } + break; + } + + mir->setVirtualRegister(vreg); + add(lir); +} + +#ifdef DEBUG +// This function checks that when making redefinitions, we don't accidentally +// coerce two incompatible types. +static inline bool IsCompatibleLIRCoercion(MIRType to, MIRType from) { + if (to == from) { + return true; + } + // In LIR, we treat boolean and int32 as the same low-level type (INTEGER). + // When snapshotting, we recover the actual JS type from MIR. + if ((to == MIRType::Int32 || to == MIRType::Boolean) && + (from == MIRType::Int32 || from == MIRType::Boolean)) { + return true; + } + // On 32-bit platforms Int32 can be redefined as IntPtr and vice versa. + // On 64-bit platforms we can redefine non-negative Int32 values as IntPtr. + if (from == MIRType::Int32 && to == MIRType::IntPtr) { + return true; + } +# ifndef JS_64BIT + if (from == MIRType::IntPtr && to == MIRType::Int32) { + return true; + } +# endif + return false; +} +#endif + +void LIRGeneratorShared::redefine(MDefinition* def, MDefinition* as) { + MOZ_ASSERT(IsCompatibleLIRCoercion(def->type(), as->type())); + + // Try to emit MIR marked as emitted-at-uses at, well, uses. For + // snapshotting reasons we delay the MIRTypes match, or when we are + // coercing between bool and int32 constants. + if (as->isEmittedAtUses() && + (def->type() == as->type() || + (as->isConstant() && + (def->type() == MIRType::Int32 || def->type() == MIRType::Boolean) && + (as->type() == MIRType::Int32 || as->type() == MIRType::Boolean)))) { + MInstruction* replacement; + if (def->type() != as->type()) { + if (as->type() == MIRType::Int32) { + replacement = + MConstant::New(alloc(), BooleanValue(as->toConstant()->toInt32())); + } else { + replacement = + MConstant::New(alloc(), Int32Value(as->toConstant()->toBoolean())); + } + def->block()->insertBefore(def->toInstruction(), replacement); + emitAtUses(replacement->toInstruction()); + } else { + replacement = as->toInstruction(); + } + def->replaceAllUsesWith(replacement); + } else { + ensureDefined(as); + def->setVirtualRegister(as->virtualRegister()); + } +} + +void LIRGeneratorShared::ensureDefined(MDefinition* mir) { + if (mir->isEmittedAtUses()) { + visitEmittedAtUses(mir->toInstruction()); + MOZ_ASSERT(mir->isLowered()); + } +} + +bool LIRGeneratorShared::willHaveDifferentLIRNodes(MDefinition* mir1, + MDefinition* mir2) { + if (mir1 != mir2) { + return true; + } + if (mir1->isEmittedAtUses()) { + return true; + } + return false; +} + +template <typename LClass, typename... Args> +LClass* LIRGeneratorShared::allocateVariadic(uint32_t numOperands, + Args&&... args) { + size_t numBytes = sizeof(LClass) + numOperands * sizeof(LAllocation); + void* buf = alloc().allocate(numBytes); + if (!buf) { + return nullptr; + } + + LClass* ins = static_cast<LClass*>(buf); + new (ins) LClass(numOperands, std::forward<Args>(args)...); + + ins->initOperandsOffset(sizeof(LClass)); + + for (uint32_t i = 0; i < numOperands; i++) { + ins->setOperand(i, LAllocation()); + } + + return ins; +} + +LUse LIRGeneratorShared::useRegister(MDefinition* mir) { + return use(mir, LUse(LUse::REGISTER)); +} + +LUse LIRGeneratorShared::useRegisterAtStart(MDefinition* mir) { + return use(mir, LUse(LUse::REGISTER, true)); +} + +LUse LIRGeneratorShared::use(MDefinition* mir) { + return use(mir, LUse(LUse::ANY)); +} + +LUse LIRGeneratorShared::useAtStart(MDefinition* mir) { + return use(mir, LUse(LUse::ANY, true)); +} + +LAllocation LIRGeneratorShared::useOrConstant(MDefinition* mir) { + if (mir->isConstant()) { + return LAllocation(mir->toConstant()); + } + return use(mir); +} + +LAllocation LIRGeneratorShared::useOrConstantAtStart(MDefinition* mir) { + if (mir->isConstant()) { + return LAllocation(mir->toConstant()); + } + return useAtStart(mir); +} + +LAllocation LIRGeneratorShared::useRegisterOrConstant(MDefinition* mir) { + if (mir->isConstant()) { + return LAllocation(mir->toConstant()); + } + return useRegister(mir); +} + +LAllocation LIRGeneratorShared::useRegisterOrConstantAtStart(MDefinition* mir) { + if (mir->isConstant()) { + return LAllocation(mir->toConstant()); + } + return useRegisterAtStart(mir); +} + +inline bool CanUseInt32Constant(MDefinition* mir) { + if (!mir->isConstant()) { + return false; + } + MConstant* cst = mir->toConstant(); + if (cst->type() == MIRType::IntPtr) { + return INT32_MIN <= cst->toIntPtr() && cst->toIntPtr() <= INT32_MAX; + } + MOZ_ASSERT(cst->type() == MIRType::Int32); + return true; +} + +LAllocation LIRGeneratorShared::useRegisterOrInt32Constant(MDefinition* mir) { + if (CanUseInt32Constant(mir)) { + return LAllocation(mir->toConstant()); + } + return useRegister(mir); +} + +LAllocation LIRGeneratorShared::useAnyOrInt32Constant(MDefinition* mir) { + if (CanUseInt32Constant(mir)) { + return LAllocation(mir->toConstant()); + } + return useAny(mir); +} + +LAllocation LIRGeneratorShared::useRegisterOrZero(MDefinition* mir) { + if (mir->isConstant() && + (mir->toConstant()->isInt32(0) || mir->toConstant()->isInt64(0))) { + return LAllocation(); + } + return useRegister(mir); +} + +LAllocation LIRGeneratorShared::useRegisterOrZeroAtStart(MDefinition* mir) { + if (mir->isConstant() && + (mir->toConstant()->isInt32(0) || mir->toConstant()->isInt64(0))) { + return LAllocation(); + } + return useRegisterAtStart(mir); +} + +LAllocation LIRGeneratorShared::useRegisterOrNonDoubleConstant( + MDefinition* mir) { + if (mir->isConstant() && mir->type() != MIRType::Double && + mir->type() != MIRType::Float32) { + return LAllocation(mir->toConstant()); + } + return useRegister(mir); +} + +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_RISCV64) +LAllocation LIRGeneratorShared::useAnyOrConstant(MDefinition* mir) { + return useRegisterOrConstant(mir); +} +LAllocation LIRGeneratorShared::useStorable(MDefinition* mir) { + return useRegister(mir); +} +LAllocation LIRGeneratorShared::useStorableAtStart(MDefinition* mir) { + return useRegisterAtStart(mir); +} + +LAllocation LIRGeneratorShared::useAny(MDefinition* mir) { + return useRegister(mir); +} +LAllocation LIRGeneratorShared::useAnyAtStart(MDefinition* mir) { + return useRegisterAtStart(mir); +} +#else +LAllocation LIRGeneratorShared::useAnyOrConstant(MDefinition* mir) { + return useOrConstant(mir); +} + +LAllocation LIRGeneratorShared::useAny(MDefinition* mir) { return use(mir); } +LAllocation LIRGeneratorShared::useAnyAtStart(MDefinition* mir) { + return useAtStart(mir); +} +LAllocation LIRGeneratorShared::useStorable(MDefinition* mir) { + return useRegisterOrConstant(mir); +} +LAllocation LIRGeneratorShared::useStorableAtStart(MDefinition* mir) { + return useRegisterOrConstantAtStart(mir); +} + +#endif + +LAllocation LIRGeneratorShared::useKeepalive(MDefinition* mir) { + return use(mir, LUse(LUse::KEEPALIVE)); +} + +LAllocation LIRGeneratorShared::useKeepaliveOrConstant(MDefinition* mir) { + if (mir->isConstant()) { + return LAllocation(mir->toConstant()); + } + return useKeepalive(mir); +} + +LUse LIRGeneratorShared::useFixed(MDefinition* mir, Register reg) { + return use(mir, LUse(reg)); +} + +LUse LIRGeneratorShared::useFixedAtStart(MDefinition* mir, Register reg) { + return use(mir, LUse(reg, true)); +} + +LUse LIRGeneratorShared::useFixed(MDefinition* mir, FloatRegister reg) { + return use(mir, LUse(reg)); +} + +LUse LIRGeneratorShared::useFixed(MDefinition* mir, AnyRegister reg) { + return reg.isFloat() ? use(mir, LUse(reg.fpu())) : use(mir, LUse(reg.gpr())); +} + +LUse LIRGeneratorShared::useFixedAtStart(MDefinition* mir, AnyRegister reg) { + return reg.isFloat() ? use(mir, LUse(reg.fpu(), true)) + : use(mir, LUse(reg.gpr(), true)); +} + +LDefinition LIRGeneratorShared::temp(LDefinition::Type type, + LDefinition::Policy policy) { + return LDefinition(getVirtualRegister(), type, policy); +} + +LInt64Definition LIRGeneratorShared::tempInt64(LDefinition::Policy policy) { +#if JS_BITS_PER_WORD == 32 + LDefinition high = temp(LDefinition::GENERAL, policy); + LDefinition low = temp(LDefinition::GENERAL, policy); + return LInt64Definition(high, low); +#else + return LInt64Definition(temp(LDefinition::GENERAL, policy)); +#endif +} + +LDefinition LIRGeneratorShared::tempFixed(Register reg) { + LDefinition t = temp(LDefinition::GENERAL); + t.setOutput(LGeneralReg(reg)); + return t; +} + +LInt64Definition LIRGeneratorShared::tempInt64Fixed(Register64 reg) { +#if JS_BITS_PER_WORD == 32 + LDefinition high = temp(LDefinition::GENERAL); + LDefinition low = temp(LDefinition::GENERAL); + high.setOutput(LGeneralReg(reg.high)); + low.setOutput(LGeneralReg(reg.low)); + return LInt64Definition(high, low); +#else + LDefinition t = temp(LDefinition::GENERAL); + t.setOutput(LGeneralReg(reg.reg)); + return LInt64Definition(t); +#endif +} + +LDefinition LIRGeneratorShared::tempFixed(FloatRegister reg) { + LDefinition t = temp(LDefinition::DOUBLE); + t.setOutput(LFloatReg(reg)); + return t; +} + +LDefinition LIRGeneratorShared::tempFloat32() { + return temp(LDefinition::FLOAT32); +} + +LDefinition LIRGeneratorShared::tempDouble() { + return temp(LDefinition::DOUBLE); +} + +#ifdef ENABLE_WASM_SIMD +LDefinition LIRGeneratorShared::tempSimd128() { + return temp(LDefinition::SIMD128); +} +#endif + +LDefinition LIRGeneratorShared::tempCopy(MDefinition* input, + uint32_t reusedInput) { + MOZ_ASSERT(input->virtualRegister()); + LDefinition t = + temp(LDefinition::TypeFrom(input->type()), LDefinition::MUST_REUSE_INPUT); + t.setReusedInput(reusedInput); + return t; +} + +template <typename T> +void LIRGeneratorShared::annotate(T* ins) { + ins->setId(lirGraph_.getInstructionId()); +} + +template <typename T> +void LIRGeneratorShared::add(T* ins, MInstruction* mir) { + MOZ_ASSERT(!ins->isPhi()); + current->add(ins); + if (mir) { + MOZ_ASSERT(current == mir->block()->lir()); + ins->setMir(mir); + } + annotate(ins); + if (ins->isCall()) { + gen->setNeedsOverrecursedCheck(); + gen->setNeedsStaticStackAlignment(); + } +} + +#ifdef JS_NUNBOX32 +// Returns the virtual register of a js::Value-defining instruction. This is +// abstracted because MBox is a special value-returning instruction that +// redefines its input payload if its input is not constant. Therefore, it is +// illegal to request a box's payload by adding VREG_DATA_OFFSET to its raw id. +static inline uint32_t VirtualRegisterOfPayload(MDefinition* mir) { + if (mir->isBox()) { + MDefinition* inner = mir->toBox()->getOperand(0); + if (!inner->isConstant() && inner->type() != MIRType::Double && + inner->type() != MIRType::Float32) { + return inner->virtualRegister(); + } + } + return mir->virtualRegister() + VREG_DATA_OFFSET; +} + +// Note: always call ensureDefined before calling useType/usePayload, +// so that emitted-at-use operands are handled correctly. +LUse LIRGeneratorShared::useType(MDefinition* mir, LUse::Policy policy) { + MOZ_ASSERT(mir->type() == MIRType::Value); + + return LUse(mir->virtualRegister() + VREG_TYPE_OFFSET, policy); +} + +LUse LIRGeneratorShared::usePayload(MDefinition* mir, LUse::Policy policy) { + MOZ_ASSERT(mir->type() == MIRType::Value); + + return LUse(VirtualRegisterOfPayload(mir), policy); +} + +LUse LIRGeneratorShared::usePayloadAtStart(MDefinition* mir, + LUse::Policy policy) { + MOZ_ASSERT(mir->type() == MIRType::Value); + + return LUse(VirtualRegisterOfPayload(mir), policy, true); +} + +LUse LIRGeneratorShared::usePayloadInRegisterAtStart(MDefinition* mir) { + return usePayloadAtStart(mir, LUse::REGISTER); +} + +void LIRGeneratorShared::fillBoxUses(LInstruction* lir, size_t n, + MDefinition* mir) { + ensureDefined(mir); + lir->getOperand(n)->toUse()->setVirtualRegister(mir->virtualRegister() + + VREG_TYPE_OFFSET); + lir->getOperand(n + 1)->toUse()->setVirtualRegister( + VirtualRegisterOfPayload(mir)); +} +#endif + +LUse LIRGeneratorShared::useRegisterForTypedLoad(MDefinition* mir, + MIRType type) { + MOZ_ASSERT(type != MIRType::Value && type != MIRType::None); + MOZ_ASSERT(mir->type() == MIRType::Object || mir->type() == MIRType::Slots); + +#ifdef JS_PUNBOX64 + // On x64, masm.loadUnboxedValue emits slightly less efficient code when + // the input and output use the same register and we're not loading an + // int32/bool/double, so we just call useRegister in this case. + if (type != MIRType::Int32 && type != MIRType::Boolean && + type != MIRType::Double) { + return useRegister(mir); + } +#endif + + return useRegisterAtStart(mir); +} + +LBoxAllocation LIRGeneratorShared::useBox(MDefinition* mir, LUse::Policy policy, + bool useAtStart) { + MOZ_ASSERT(mir->type() == MIRType::Value); + + ensureDefined(mir); + +#if defined(JS_NUNBOX32) + return LBoxAllocation( + LUse(mir->virtualRegister(), policy, useAtStart), + LUse(VirtualRegisterOfPayload(mir), policy, useAtStart)); +#else + return LBoxAllocation(LUse(mir->virtualRegister(), policy, useAtStart)); +#endif +} + +LBoxAllocation LIRGeneratorShared::useBoxOrTyped(MDefinition* mir, + bool useAtStart) { + if (mir->type() == MIRType::Value) { + return useBox(mir, LUse::REGISTER, useAtStart); + } + +#if defined(JS_NUNBOX32) + return LBoxAllocation(useAtStart ? useRegisterAtStart(mir) : useRegister(mir), + LAllocation()); +#else + return LBoxAllocation(useAtStart ? useRegisterAtStart(mir) + : useRegister(mir)); +#endif +} + +LBoxAllocation LIRGeneratorShared::useBoxOrTypedOrConstant(MDefinition* mir, + bool useConstant, + bool useAtStart) { + if (useConstant && mir->isConstant()) { +#if defined(JS_NUNBOX32) + return LBoxAllocation(LAllocation(mir->toConstant()), LAllocation()); +#else + return LBoxAllocation(LAllocation(mir->toConstant())); +#endif + } + + return useBoxOrTyped(mir, useAtStart); +} + +LInt64Allocation LIRGeneratorShared::useInt64(MDefinition* mir, + LUse::Policy policy, + bool useAtStart) { + MOZ_ASSERT(mir->type() == MIRType::Int64); + + ensureDefined(mir); + + uint32_t vreg = mir->virtualRegister(); +#if JS_BITS_PER_WORD == 32 + return LInt64Allocation(LUse(vreg + INT64HIGH_INDEX, policy, useAtStart), + LUse(vreg + INT64LOW_INDEX, policy, useAtStart)); +#else + return LInt64Allocation(LUse(vreg, policy, useAtStart)); +#endif +} + +LInt64Allocation LIRGeneratorShared::useInt64Fixed(MDefinition* mir, + Register64 regs, + bool useAtStart) { + MOZ_ASSERT(mir->type() == MIRType::Int64); + + ensureDefined(mir); + + uint32_t vreg = mir->virtualRegister(); +#if JS_BITS_PER_WORD == 32 + return LInt64Allocation(LUse(regs.high, vreg + INT64HIGH_INDEX, useAtStart), + LUse(regs.low, vreg + INT64LOW_INDEX, useAtStart)); +#else + return LInt64Allocation(LUse(regs.reg, vreg, useAtStart)); +#endif +} + +LInt64Allocation LIRGeneratorShared::useInt64FixedAtStart(MDefinition* mir, + Register64 regs) { + return useInt64Fixed(mir, regs, true); +} + +LInt64Allocation LIRGeneratorShared::useInt64(MDefinition* mir, + bool useAtStart) { + // On 32-bit platforms, always load the value in registers. +#if JS_BITS_PER_WORD == 32 + return useInt64(mir, LUse::REGISTER, useAtStart); +#else + return useInt64(mir, LUse::ANY, useAtStart); +#endif +} + +LInt64Allocation LIRGeneratorShared::useInt64AtStart(MDefinition* mir) { + return useInt64(mir, /* useAtStart = */ true); +} + +LInt64Allocation LIRGeneratorShared::useInt64Register(MDefinition* mir, + bool useAtStart) { + return useInt64(mir, LUse::REGISTER, useAtStart); +} + +LInt64Allocation LIRGeneratorShared::useInt64OrConstant(MDefinition* mir, + bool useAtStart) { + if (mir->isConstant()) { +#if defined(JS_NUNBOX32) + return LInt64Allocation(LAllocation(mir->toConstant()), LAllocation()); +#else + return LInt64Allocation(LAllocation(mir->toConstant())); +#endif + } + return useInt64(mir, useAtStart); +} + +LInt64Allocation LIRGeneratorShared::useInt64RegisterOrConstant( + MDefinition* mir, bool useAtStart) { + if (mir->isConstant()) { +#if defined(JS_NUNBOX32) + return LInt64Allocation(LAllocation(mir->toConstant()), LAllocation()); +#else + return LInt64Allocation(LAllocation(mir->toConstant())); +#endif + } + return useInt64Register(mir, useAtStart); +} + +LInt64Allocation LIRGeneratorShared::useInt64RegisterAtStart(MDefinition* mir) { + return useInt64Register(mir, /* useAtStart = */ true); +} + +LInt64Allocation LIRGeneratorShared::useInt64RegisterOrConstantAtStart( + MDefinition* mir) { + return useInt64RegisterOrConstant(mir, /* useAtStart = */ true); +} + +LInt64Allocation LIRGeneratorShared::useInt64OrConstantAtStart( + MDefinition* mir) { + return useInt64OrConstant(mir, /* useAtStart = */ true); +} + +void LIRGeneratorShared::lowerConstantDouble(double d, MInstruction* mir) { + define(new (alloc()) LDouble(d), mir); +} +void LIRGeneratorShared::lowerConstantFloat32(float f, MInstruction* mir) { + define(new (alloc()) LFloat32(f), mir); +} + +} // namespace jit +} // namespace js + +#endif /* jit_shared_Lowering_shared_inl_h */ diff --git a/js/src/jit/shared/Lowering-shared.cpp b/js/src/jit/shared/Lowering-shared.cpp new file mode 100644 index 0000000000..754cbe71e7 --- /dev/null +++ b/js/src/jit/shared/Lowering-shared.cpp @@ -0,0 +1,319 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/shared/Lowering-shared-inl.h" + +#include "jit/LIR.h" +#include "jit/Lowering.h" +#include "jit/MIR.h" +#include "jit/ScalarTypeUtils.h" + +#include "vm/SymbolType.h" + +using namespace js; +using namespace jit; + +using mozilla::Maybe; +using mozilla::Nothing; +using mozilla::Some; + +bool LIRGeneratorShared::ShouldReorderCommutative(MDefinition* lhs, + MDefinition* rhs, + MInstruction* ins) { + // lhs and rhs are used by the commutative operator. + MOZ_ASSERT(lhs->hasDefUses()); + MOZ_ASSERT(rhs->hasDefUses()); + + // Ensure that if there is a constant, then it is in rhs. + if (rhs->isConstant()) { + return false; + } + if (lhs->isConstant()) { + return true; + } + + // Since clobbering binary operations clobber the left operand, prefer a + // non-constant lhs operand with no further uses. To be fully precise, we + // should check whether this is the *last* use, but checking hasOneDefUse() + // is a decent approximation which doesn't require any extra analysis. + bool rhsSingleUse = rhs->hasOneDefUse(); + bool lhsSingleUse = lhs->hasOneDefUse(); + if (rhsSingleUse) { + if (!lhsSingleUse) { + return true; + } + } else { + if (lhsSingleUse) { + return false; + } + } + + // If this is a reduction-style computation, such as + // + // sum = 0; + // for (...) + // sum += ...; + // + // put the phi on the left to promote coalescing. This is fairly specific. + if (rhsSingleUse && rhs->isPhi() && rhs->block()->isLoopHeader() && + ins == rhs->toPhi()->getLoopBackedgeOperand()) { + return true; + } + + return false; +} + +void LIRGeneratorShared::ReorderCommutative(MDefinition** lhsp, + MDefinition** rhsp, + MInstruction* ins) { + MDefinition* lhs = *lhsp; + MDefinition* rhs = *rhsp; + + if (ShouldReorderCommutative(lhs, rhs, ins)) { + *rhsp = lhs; + *lhsp = rhs; + } +} + +void LIRGeneratorShared::definePhiOneRegister(MPhi* phi, size_t lirIndex) { + LPhi* lir = current->getPhi(lirIndex); + + uint32_t vreg = getVirtualRegister(); + + phi->setVirtualRegister(vreg); + lir->setDef(0, LDefinition(vreg, LDefinition::TypeFrom(phi->type()))); + annotate(lir); +} + +#ifdef JS_NUNBOX32 +void LIRGeneratorShared::definePhiTwoRegisters(MPhi* phi, size_t lirIndex) { + LPhi* type = current->getPhi(lirIndex + VREG_TYPE_OFFSET); + LPhi* payload = current->getPhi(lirIndex + VREG_DATA_OFFSET); + + uint32_t typeVreg = getVirtualRegister(); + phi->setVirtualRegister(typeVreg); + + uint32_t payloadVreg = getVirtualRegister(); + MOZ_ASSERT_IF(!errored(), typeVreg + 1 == payloadVreg); + + type->setDef(0, LDefinition(typeVreg, LDefinition::TYPE)); + payload->setDef(0, LDefinition(payloadVreg, LDefinition::PAYLOAD)); + annotate(type); + annotate(payload); +} +#endif + +void LIRGeneratorShared::lowerTypedPhiInput(MPhi* phi, uint32_t inputPosition, + LBlock* block, size_t lirIndex) { + MDefinition* operand = phi->getOperand(inputPosition); + LPhi* lir = block->getPhi(lirIndex); + lir->setOperand(inputPosition, LUse(operand->virtualRegister(), LUse::ANY)); +} + +LRecoverInfo* LIRGeneratorShared::getRecoverInfo(MResumePoint* rp) { + if (cachedRecoverInfo_ && cachedRecoverInfo_->mir() == rp) { + return cachedRecoverInfo_; + } + + LRecoverInfo* recoverInfo = LRecoverInfo::New(gen, rp); + if (!recoverInfo) { + return nullptr; + } + + cachedRecoverInfo_ = recoverInfo; + return recoverInfo; +} + +#ifdef DEBUG +bool LRecoverInfo::OperandIter::canOptimizeOutIfUnused() { + MDefinition* ins = **this; + + // We check ins->type() in addition to ins->isUnused() because + // EliminateDeadResumePointOperands may replace nodes with the constant + // MagicValue(JS_OPTIMIZED_OUT). + if ((ins->isUnused() || ins->type() == MIRType::MagicOptimizedOut) && + (*it_)->isResumePoint()) { + return !(*it_)->toResumePoint()->isObservableOperand(op_); + } + + return true; +} +#endif + +LAllocation LIRGeneratorShared::useRegisterOrIndexConstant( + MDefinition* mir, Scalar::Type type, int32_t offsetAdjustment) { + if (CanUseInt32Constant(mir)) { + MConstant* cst = mir->toConstant(); + int32_t val = + cst->type() == MIRType::Int32 ? cst->toInt32() : cst->toIntPtr(); + int32_t offset; + if (ArrayOffsetFitsInInt32(val, type, offsetAdjustment, &offset)) { + return LAllocation(mir->toConstant()); + } + } + return useRegister(mir); +} + +#ifdef JS_NUNBOX32 +LSnapshot* LIRGeneratorShared::buildSnapshot(MResumePoint* rp, + BailoutKind kind) { + LRecoverInfo* recoverInfo = getRecoverInfo(rp); + if (!recoverInfo) { + return nullptr; + } + + LSnapshot* snapshot = LSnapshot::New(gen, recoverInfo, kind); + if (!snapshot) { + return nullptr; + } + + size_t index = 0; + for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) { + // Check that optimized out operands are in eliminable slots. + MOZ_ASSERT(it.canOptimizeOutIfUnused()); + + MDefinition* ins = *it; + + if (ins->isRecoveredOnBailout()) { + continue; + } + + LAllocation* type = snapshot->typeOfSlot(index); + LAllocation* payload = snapshot->payloadOfSlot(index); + ++index; + + if (ins->isBox()) { + ins = ins->toBox()->getOperand(0); + } + + // Guards should never be eliminated. + MOZ_ASSERT_IF(ins->isUnused(), !ins->isGuard()); + + // Snapshot operands other than constants should never be + // emitted-at-uses. Try-catch support depends on there being no + // code between an instruction and the LOsiPoint that follows it. + MOZ_ASSERT_IF(!ins->isConstant(), !ins->isEmittedAtUses()); + + // The register allocation will fill these fields in with actual + // register/stack assignments. During code generation, we can restore + // interpreter state with the given information. Note that for + // constants, including known types, we record a dummy placeholder, + // since we can recover the same information, much cleaner, from MIR. + if (ins->isConstant() || ins->isUnused()) { + *type = LAllocation(); + *payload = LAllocation(); + } else if (ins->type() != MIRType::Value) { + *type = LAllocation(); + *payload = use(ins, LUse(LUse::KEEPALIVE)); + } else { + *type = useType(ins, LUse::KEEPALIVE); + *payload = usePayload(ins, LUse::KEEPALIVE); + } + } + + return snapshot; +} + +#elif JS_PUNBOX64 + +LSnapshot* LIRGeneratorShared::buildSnapshot(MResumePoint* rp, + BailoutKind kind) { + LRecoverInfo* recoverInfo = getRecoverInfo(rp); + if (!recoverInfo) { + return nullptr; + } + + LSnapshot* snapshot = LSnapshot::New(gen, recoverInfo, kind); + if (!snapshot) { + return nullptr; + } + + size_t index = 0; + for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) { + // Check that optimized out operands are in eliminable slots. + MOZ_ASSERT(it.canOptimizeOutIfUnused()); + + MDefinition* def = *it; + + if (def->isRecoveredOnBailout()) { + continue; + } + + if (def->isBox()) { + def = def->toBox()->getOperand(0); + } + + // Guards should never be eliminated. + MOZ_ASSERT_IF(def->isUnused(), !def->isGuard()); + + // Snapshot operands other than constants should never be + // emitted-at-uses. Try-catch support depends on there being no + // code between an instruction and the LOsiPoint that follows it. + MOZ_ASSERT_IF(!def->isConstant(), !def->isEmittedAtUses()); + + LAllocation* a = snapshot->getEntry(index++); + + if (def->isUnused()) { + *a = LAllocation(); + continue; + } + + *a = useKeepaliveOrConstant(def); + } + + return snapshot; +} +#endif + +void LIRGeneratorShared::assignSnapshot(LInstruction* ins, BailoutKind kind) { + // assignSnapshot must be called before define/add, since + // it may add new instructions for emitted-at-use operands. + MOZ_ASSERT(ins->id() == 0); + MOZ_ASSERT(kind != BailoutKind::Unknown); + + LSnapshot* snapshot = buildSnapshot(lastResumePoint_, kind); + if (!snapshot) { + abort(AbortReason::Alloc, "buildSnapshot failed"); + return; + } + + ins->assignSnapshot(snapshot); +} + +void LIRGeneratorShared::assignSafepoint(LInstruction* ins, MInstruction* mir, + BailoutKind kind) { + MOZ_ASSERT(!osiPoint_); + MOZ_ASSERT(!ins->safepoint()); + + ins->initSafepoint(alloc()); + + MResumePoint* mrp = + mir->resumePoint() ? mir->resumePoint() : lastResumePoint_; + LSnapshot* postSnapshot = buildSnapshot(mrp, kind); + if (!postSnapshot) { + abort(AbortReason::Alloc, "buildSnapshot failed"); + return; + } + + osiPoint_ = new (alloc()) LOsiPoint(ins->safepoint(), postSnapshot); + + if (!lirGraph_.noteNeedsSafepoint(ins)) { + abort(AbortReason::Alloc, "noteNeedsSafepoint failed"); + return; + } +} + +void LIRGeneratorShared::assignWasmSafepoint(LInstruction* ins) { + MOZ_ASSERT(!osiPoint_); + MOZ_ASSERT(!ins->safepoint()); + + ins->initSafepoint(alloc()); + + if (!lirGraph_.noteNeedsSafepoint(ins)) { + abort(AbortReason::Alloc, "noteNeedsSafepoint failed"); + return; + } +} diff --git a/js/src/jit/shared/Lowering-shared.h b/js/src/jit/shared/Lowering-shared.h new file mode 100644 index 0000000000..d26e349d8c --- /dev/null +++ b/js/src/jit/shared/Lowering-shared.h @@ -0,0 +1,371 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_shared_Lowering_shared_h +#define jit_shared_Lowering_shared_h + +// This file declares the structures that are used for attaching LIR to a +// MIRGraph. + +#include "jit/LIR.h" +#include "jit/MIRGenerator.h" + +namespace js { +namespace jit { + +class MIRGenerator; +class MIRGraph; +class MDefinition; +class MInstruction; +class LOsiPoint; + +class LIRGeneratorShared { + protected: + MIRGenerator* gen; + MIRGraph& graph; + LIRGraph& lirGraph_; + LBlock* current; + MResumePoint* lastResumePoint_; + LRecoverInfo* cachedRecoverInfo_; + LOsiPoint* osiPoint_; + + LIRGeneratorShared(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph) + : gen(gen), + graph(graph), + lirGraph_(lirGraph), + current(nullptr), + lastResumePoint_(nullptr), + cachedRecoverInfo_(nullptr), + osiPoint_(nullptr) {} + + MIRGenerator* mir() { return gen; } + + // Abort errors are caught at end of visitInstruction. It is possible for + // multiple errors to be detected before the end of visitInstruction. In + // this case, we only report the first back to the MIRGenerator. + bool errored() { return gen->getOffThreadStatus().isErr(); } + void abort(AbortReason r, const char* message, ...) MOZ_FORMAT_PRINTF(3, 4) { + if (errored()) { + return; + } + + va_list ap; + va_start(ap, message); + auto reason_ = gen->abortFmt(r, message, ap); + va_end(ap); + gen->setOffThreadStatus(reason_); + } + void abort(AbortReason r) { + if (errored()) { + return; + } + + auto reason_ = gen->abort(r); + gen->setOffThreadStatus(reason_); + } + + static void ReorderCommutative(MDefinition** lhsp, MDefinition** rhsp, + MInstruction* ins); + static bool ShouldReorderCommutative(MDefinition* lhs, MDefinition* rhs, + MInstruction* ins); + + // A backend can decide that an instruction should be emitted at its uses, + // rather than at its definition. To communicate this, set the + // instruction's virtual register set to 0. When using the instruction, + // its virtual register is temporarily reassigned. To know to clear it + // after constructing the use information, the worklist bit is temporarily + // unset. + // + // The backend can use the worklist bit to determine whether or not a + // definition should be created. + inline void emitAtUses(MInstruction* mir); + + // The lowest-level calls to use, those that do not wrap another call to + // use(), must prefix grabbing virtual register IDs by these calls. + inline void ensureDefined(MDefinition* mir); + + void visitEmittedAtUses(MInstruction* ins); + + // These all create a use of a virtual register, with an optional + // allocation policy. + // + // Some of these use functions have atStart variants. + // - non-atStart variants will tell the register allocator that the input + // allocation must be different from any Temp or Definition also needed for + // this LInstruction. + // - atStart variants relax that restriction and allow the input to be in + // the same register as any output Definition (but not Temps) used by the + // LInstruction. Note that it doesn't *imply* this will actually happen, + // but gives a hint to the register allocator that it can do it. + // + // TL;DR: Use non-atStart variants only if you need the input value after + // writing to any definitions (excluding temps), during code generation of + // this LInstruction. Otherwise, use atStart variants, which will lower + // register pressure. + // + // There is an additional constraint. Consider a MIR node with two + // MDefinition* operands, op1 and op2. If the node reuses the register of op1 + // for its output then op1 must be used as atStart. Then, if op1 and op2 + // represent the same LIR node then op2 must be an atStart use too; otherwise + // op2 must be a non-atStart use. There is however not always a 1-1 mapping + // from MDefinition* to LNode*, so to determine whether two MDefinition* map + // to the same LNode*, ALWAYS go via the willHaveDifferentLIRNodes() + // predicate. Do not use pointer equality on the MIR nodes. + // + // Do not add other conditions when using willHaveDifferentLIRNodes(). The + // predicate is the source of truth about whether to use atStart or not, no + // other conditions may apply in contexts when it is appropriate to use it. + inline LUse use(MDefinition* mir, LUse policy); + inline LUse use(MDefinition* mir); + inline LUse useAtStart(MDefinition* mir); + inline LUse useRegister(MDefinition* mir); + inline LUse useRegisterAtStart(MDefinition* mir); + inline LUse useFixed(MDefinition* mir, Register reg); + inline LUse useFixed(MDefinition* mir, FloatRegister reg); + inline LUse useFixed(MDefinition* mir, AnyRegister reg); + inline LUse useFixedAtStart(MDefinition* mir, Register reg); + inline LUse useFixedAtStart(MDefinition* mir, AnyRegister reg); + inline LAllocation useOrConstant(MDefinition* mir); + inline LAllocation useOrConstantAtStart(MDefinition* mir); + // "Any" is architecture dependent, and will include registers and stack + // slots on X86, and only registers on ARM. + inline LAllocation useAny(MDefinition* mir); + inline LAllocation useAnyAtStart(MDefinition* mir); + inline LAllocation useAnyOrConstant(MDefinition* mir); + // "Storable" is architecture dependend, and will include registers and + // constants on X86 and only registers on ARM. This is a generic "things + // we can expect to write into memory in 1 instruction". + inline LAllocation useStorable(MDefinition* mir); + inline LAllocation useStorableAtStart(MDefinition* mir); + inline LAllocation useKeepalive(MDefinition* mir); + inline LAllocation useKeepaliveOrConstant(MDefinition* mir); + inline LAllocation useRegisterOrConstant(MDefinition* mir); + inline LAllocation useRegisterOrConstantAtStart(MDefinition* mir); + inline LAllocation useRegisterOrZeroAtStart(MDefinition* mir); + inline LAllocation useRegisterOrZero(MDefinition* mir); + inline LAllocation useRegisterOrNonDoubleConstant(MDefinition* mir); + + // These methods accept either an Int32 or IntPtr value. A constant is used if + // the value fits in an int32. + inline LAllocation useRegisterOrInt32Constant(MDefinition* mir); + inline LAllocation useAnyOrInt32Constant(MDefinition* mir); + + // Like useRegisterOrInt32Constant, but uses a constant only if + // |int32val * Scalar::byteSize(type) + offsetAdjustment| doesn't overflow + // int32. + LAllocation useRegisterOrIndexConstant(MDefinition* mir, Scalar::Type type, + int32_t offsetAdjustment = 0); + + inline LUse useRegisterForTypedLoad(MDefinition* mir, MIRType type); + +#ifdef JS_NUNBOX32 + inline LUse useType(MDefinition* mir, LUse::Policy policy); + inline LUse usePayload(MDefinition* mir, LUse::Policy policy); + inline LUse usePayloadAtStart(MDefinition* mir, LUse::Policy policy); + inline LUse usePayloadInRegisterAtStart(MDefinition* mir); + + // Adds a box input to an instruction, setting operand |n| to the type and + // |n+1| to the payload. Does not modify the operands, instead expecting a + // policy to already be set. + inline void fillBoxUses(LInstruction* lir, size_t n, MDefinition* mir); +#endif + + // Test whether mir1 and mir2 may give rise to different LIR nodes even if + // mir1 == mir2; use it to guide the selection of the use directive for one of + // the nodes in the context of a reused input. See comments above about why + // it's important to use this predicate and not pointer equality. + // + // This predicate may be called before or after the application of a use + // directive to the first of the nodes, but it is meaningless to call it after + // the application of a directive to the second node. + inline bool willHaveDifferentLIRNodes(MDefinition* mir1, MDefinition* mir2); + + // These create temporary register requests. + inline LDefinition temp(LDefinition::Type type = LDefinition::GENERAL, + LDefinition::Policy policy = LDefinition::REGISTER); + inline LInt64Definition tempInt64( + LDefinition::Policy policy = LDefinition::REGISTER); + inline LDefinition tempFloat32(); + inline LDefinition tempDouble(); +#ifdef ENABLE_WASM_SIMD + inline LDefinition tempSimd128(); +#endif + inline LDefinition tempCopy(MDefinition* input, uint32_t reusedInput); + + // Note that the fixed register has a GENERAL type, + // unless the arg is of FloatRegister type + inline LDefinition tempFixed(Register reg); + inline LDefinition tempFixed(FloatRegister reg); + inline LInt64Definition tempInt64Fixed(Register64 reg); + + template <size_t Ops, size_t Temps> + inline void defineFixed(LInstructionHelper<1, Ops, Temps>* lir, + MDefinition* mir, const LAllocation& output); + + template <size_t Temps> + inline void defineBox( + details::LInstructionFixedDefsTempsHelper<BOX_PIECES, Temps>* lir, + MDefinition* mir, LDefinition::Policy policy = LDefinition::REGISTER); + + template <size_t Ops, size_t Temps> + inline void defineInt64(LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, + MDefinition* mir, + LDefinition::Policy policy = LDefinition::REGISTER); + + template <size_t Ops, size_t Temps> + inline void defineInt64Fixed( + LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir, + const LInt64Allocation& output); + + inline void defineReturn(LInstruction* lir, MDefinition* mir); + + template <size_t X> + inline void define(details::LInstructionFixedDefsTempsHelper<1, X>* lir, + MDefinition* mir, + LDefinition::Policy policy = LDefinition::REGISTER); + template <size_t X> + inline void define(details::LInstructionFixedDefsTempsHelper<1, X>* lir, + MDefinition* mir, const LDefinition& def); + + template <size_t Ops, size_t Temps> + inline void defineReuseInput(LInstructionHelper<1, Ops, Temps>* lir, + MDefinition* mir, uint32_t operand); + + template <size_t Ops, size_t Temps> + inline void defineBoxReuseInput( + LInstructionHelper<BOX_PIECES, Ops, Temps>* lir, MDefinition* mir, + uint32_t operand); + + template <size_t Ops, size_t Temps> + inline void defineInt64ReuseInput( + LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir, + uint32_t operand); + + // Returns a box allocation for a Value-typed instruction. + inline LBoxAllocation useBox(MDefinition* mir, + LUse::Policy policy = LUse::REGISTER, + bool useAtStart = false); + + // Returns a box allocation. The use is either typed, a Value, or + // a constant (if useConstant is true). + inline LBoxAllocation useBoxOrTypedOrConstant(MDefinition* mir, + bool useConstant, + bool useAtStart = false); + inline LBoxAllocation useBoxOrTyped(MDefinition* mir, + bool useAtStart = false); + + // Returns an int64 allocation for an Int64-typed instruction. + inline LInt64Allocation useInt64(MDefinition* mir, LUse::Policy policy, + bool useAtStart); + inline LInt64Allocation useInt64(MDefinition* mir, bool useAtStart = false); + inline LInt64Allocation useInt64AtStart(MDefinition* mir); + inline LInt64Allocation useInt64OrConstant(MDefinition* mir, + bool useAtStart = false); + inline LInt64Allocation useInt64Register(MDefinition* mir, + bool useAtStart = false); + inline LInt64Allocation useInt64RegisterOrConstant(MDefinition* mir, + bool useAtStart = false); + inline LInt64Allocation useInt64Fixed(MDefinition* mir, Register64 regs, + bool useAtStart = false); + inline LInt64Allocation useInt64FixedAtStart(MDefinition* mir, + Register64 regs); + + inline LInt64Allocation useInt64RegisterAtStart(MDefinition* mir); + inline LInt64Allocation useInt64RegisterOrConstantAtStart(MDefinition* mir); + inline LInt64Allocation useInt64OrConstantAtStart(MDefinition* mir); + + // Rather than defining a new virtual register, sets |ins| to have the same + // virtual register as |as|. + inline void redefine(MDefinition* ins, MDefinition* as); + + template <typename LClass, typename... Args> + inline LClass* allocateVariadic(uint32_t numOperands, Args&&... args); + + TempAllocator& alloc() const { return graph.alloc(); } + + uint32_t getVirtualRegister() { + uint32_t vreg = lirGraph_.getVirtualRegister(); + + // If we run out of virtual registers, mark code generation as having + // failed and return a dummy vreg. Include a + 1 here for NUNBOX32 + // platforms that expect Value vregs to be adjacent. + if (vreg + 1 >= MAX_VIRTUAL_REGISTERS) { + abort(AbortReason::Alloc, "max virtual registers"); + return 1; + } + return vreg; + } + + template <typename T> + void annotate(T* ins); + template <typename T> + void add(T* ins, MInstruction* mir = nullptr); + + void lowerTypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block, + size_t lirIndex); + + void definePhiOneRegister(MPhi* phi, size_t lirIndex); +#ifdef JS_NUNBOX32 + void definePhiTwoRegisters(MPhi* phi, size_t lirIndex); +#endif + + void defineTypedPhi(MPhi* phi, size_t lirIndex) { + // One register containing the payload. + definePhiOneRegister(phi, lirIndex); + } + void defineUntypedPhi(MPhi* phi, size_t lirIndex) { +#ifdef JS_NUNBOX32 + // Two registers: one for the type, one for the payload. + definePhiTwoRegisters(phi, lirIndex); +#else + // One register containing the full Value. + definePhiOneRegister(phi, lirIndex); +#endif + } + + LOsiPoint* popOsiPoint() { + LOsiPoint* tmp = osiPoint_; + osiPoint_ = nullptr; + return tmp; + } + + LRecoverInfo* getRecoverInfo(MResumePoint* rp); + LSnapshot* buildSnapshot(MResumePoint* rp, BailoutKind kind); + bool assignPostSnapshot(MInstruction* mir, LInstruction* ins); + + // Marks this instruction as fallible, meaning that before it performs + // effects (if any), it may check pre-conditions and bailout if they do not + // hold. This function informs the register allocator that it will need to + // capture appropriate state. + void assignSnapshot(LInstruction* ins, BailoutKind kind); + + // Marks this instruction as needing to call into either the VM or GC. This + // function may build a snapshot that captures the result of its own + // instruction, and as such, should generally be called after define*(). + void assignSafepoint(LInstruction* ins, MInstruction* mir, + BailoutKind kind = BailoutKind::DuringVMCall); + + // Marks this instruction as needing a wasm safepoint. + void assignWasmSafepoint(LInstruction* ins); + + inline void lowerConstantDouble(double d, MInstruction* mir); + inline void lowerConstantFloat32(float f, MInstruction* mir); + + bool canSpecializeWasmCompareAndSelect(MCompare::CompareType compTy, + MIRType insTy); + void lowerWasmCompareAndSelect(MWasmSelect* ins, MDefinition* lhs, + MDefinition* rhs, MCompare::CompareType compTy, + JSOp jsop); + + public: + // Whether to generate typed reads for element accesses with hole checks. + static bool allowTypedElementHoleCheck() { return false; } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_shared_Lowering_shared_h */ |