diff options
Diffstat (limited to 'js/src/jit/RegisterSets.h')
| -rw-r--r-- | js/src/jit/RegisterSets.h | 1332 |
1 files changed, 1332 insertions, 0 deletions
diff --git a/js/src/jit/RegisterSets.h b/js/src/jit/RegisterSets.h new file mode 100644 index 0000000000..9ffc48f651 --- /dev/null +++ b/js/src/jit/RegisterSets.h @@ -0,0 +1,1332 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_RegisterSets_h +#define jit_RegisterSets_h + +#include "mozilla/Assertions.h" +#include "mozilla/Attributes.h" +#include "mozilla/Variant.h" + +#include <new> +#include <stddef.h> +#include <stdint.h> + +#include "jit/IonTypes.h" +#include "jit/Registers.h" +#include "js/Value.h" + +namespace js { +namespace jit { + +struct AnyRegister { + using Code = uint8_t; + + static const uint8_t Total = Registers::Total + FloatRegisters::Total; + static const uint8_t FirstFloatReg = Registers::Total; + static const uint8_t Invalid = UINT8_MAX; + + static_assert(size_t(Registers::Total) + FloatRegisters::Total <= UINT8_MAX, + "Number of registers must fit in uint8_t"); + + private: + Code code_; + + public: + AnyRegister() : code_(Invalid) {} + + explicit AnyRegister(Register gpr) { code_ = gpr.code(); } + explicit AnyRegister(FloatRegister fpu) { + code_ = fpu.code() + Registers::Total; + } + static AnyRegister FromCode(uint8_t i) { + MOZ_ASSERT(i < Total); + AnyRegister r; + r.code_ = i; + return r; + } + bool isFloat() const { + MOZ_ASSERT(isValid()); + return code_ >= Registers::Total; + } + Register gpr() const { + MOZ_ASSERT(!isFloat()); + return Register::FromCode(code_); + } + FloatRegister fpu() const { + MOZ_ASSERT(isFloat()); + return FloatRegister::FromCode(code_ - Registers::Total); + } + bool operator==(AnyRegister other) const { + // We don't need the operands to be valid to test for equality. + return code_ == other.code_; + } + bool operator!=(AnyRegister other) const { + // We don't need the operands to be valid to test for equality. + return code_ != other.code_; + } + const char* name() const { return isFloat() ? fpu().name() : gpr().name(); } + Code code() const { + MOZ_ASSERT(isValid()); + return code_; + } + bool volatile_() const { + return isFloat() ? fpu().volatile_() : gpr().volatile_(); + } + AnyRegister aliased(uint8_t aliasIdx) const { + AnyRegister ret; + if (isFloat()) { + ret = AnyRegister(fpu().aliased(aliasIdx)); + } else { + ret = AnyRegister(gpr().aliased(aliasIdx)); + } + MOZ_ASSERT_IF(aliasIdx == 0, ret == *this); + return ret; + } + uint8_t numAliased() const { + if (isFloat()) { + return fpu().numAliased(); + } + return gpr().numAliased(); + } + bool aliases(const AnyRegister& other) const { + if (isFloat() && other.isFloat()) { + return fpu().aliases(other.fpu()); + } + if (!isFloat() && !other.isFloat()) { + return gpr().aliases(other.gpr()); + } + return false; + } + // do the two registers hold the same type of data (e.g. both float32, both + // gpr) + bool isCompatibleReg(const AnyRegister other) const { + if (isFloat() && other.isFloat()) { + return fpu().equiv(other.fpu()); + } + if (!isFloat() && !other.isFloat()) { + return true; + } + return false; + } + bool isValid() const { return code_ != Invalid; } +}; + +// Registers to hold a boxed value. Uses one register on 64 bit +// platforms, two registers on 32 bit platforms. +class ValueOperand { +#if defined(JS_NUNBOX32) + Register type_; + Register payload_; + + public: + constexpr ValueOperand(Register type, Register payload) + : type_(type), payload_(payload) {} + + constexpr Register typeReg() const { return type_; } + constexpr Register payloadReg() const { return payload_; } + constexpr Register64 toRegister64() const { + return Register64(typeReg(), payloadReg()); + } + constexpr bool aliases(Register reg) const { + return type_ == reg || payload_ == reg; + } + constexpr Register payloadOrValueReg() const { return payloadReg(); } + bool hasVolatileReg() const { + return type_.volatile_() || payload_.volatile_(); + } + constexpr bool operator==(const ValueOperand& o) const { + return type_ == o.type_ && payload_ == o.payload_; + } + constexpr bool operator!=(const ValueOperand& o) const { + return !(*this == o); + } + +#elif defined(JS_PUNBOX64) + Register value_; + + public: + explicit constexpr ValueOperand(Register value) : value_(value) {} + + constexpr Register valueReg() const { return value_; } + constexpr Register64 toRegister64() const { return Register64(valueReg()); } + constexpr bool aliases(Register reg) const { return value_ == reg; } + constexpr Register payloadOrValueReg() const { return valueReg(); } + bool hasVolatileReg() const { return value_.volatile_(); } + constexpr bool operator==(const ValueOperand& o) const { + return value_ == o.value_; + } + constexpr bool operator!=(const ValueOperand& o) const { + return !(*this == o); + } +#endif + + constexpr Register scratchReg() const { return payloadOrValueReg(); } + + ValueOperand() = default; +}; + +// Registers to hold either either a typed or untyped value. +class TypedOrValueRegister { + // Type of value being stored. + MIRType type_; + + union U { + AnyRegister::Code typed; +#if defined(JS_PUNBOX64) + Register::Code value; +#elif defined(JS_NUNBOX32) + struct { + Register::Code valueType; + Register::Code valuePayload; + } s; +#else +# error "Bad architecture" +#endif + } data; + + public: + TypedOrValueRegister() = default; + + TypedOrValueRegister(MIRType type, AnyRegister reg) : type_(type) { + data.typed = reg.code(); + } + + MOZ_IMPLICIT TypedOrValueRegister(ValueOperand value) + : type_(MIRType::Value) { +#if defined(JS_PUNBOX64) + data.value = value.valueReg().code(); +#elif defined(JS_NUNBOX32) + data.s.valueType = value.typeReg().code(); + data.s.valuePayload = value.payloadReg().code(); +#else +# error "Bad architecture" +#endif + } + + MIRType type() const { return type_; } + + bool hasTyped() const { + return type() != MIRType::None && type() != MIRType::Value; + } + + bool hasValue() const { return type() == MIRType::Value; } + + AnyRegister typedReg() const { + MOZ_ASSERT(hasTyped()); + return AnyRegister::FromCode(data.typed); + } + + ValueOperand valueReg() const { + MOZ_ASSERT(hasValue()); +#if defined(JS_PUNBOX64) + return ValueOperand(Register::FromCode(data.value)); +#elif defined(JS_NUNBOX32) + return ValueOperand(Register::FromCode(data.s.valueType), + Register::FromCode(data.s.valuePayload)); +#else +# error "Bad architecture" +#endif + } + + AnyRegister scratchReg() { + if (hasValue()) { + return AnyRegister(valueReg().scratchReg()); + } + return typedReg(); + } +}; + +// A constant value, or registers to hold a typed/untyped value. +class ConstantOrRegister { + // Whether a constant value is being stored. + bool constant_; + + // Space to hold either a Value or a TypedOrValueRegister. + union U { + JS::Value constant; + TypedOrValueRegister reg; + + // |constant| has a non-trivial constructor and therefore MUST be + // placement-new'd into existence. + MOZ_PUSH_DISABLE_NONTRIVIAL_UNION_WARNINGS + U() {} + MOZ_POP_DISABLE_NONTRIVIAL_UNION_WARNINGS + } data; + + public: + ConstantOrRegister() = delete; + + MOZ_IMPLICIT ConstantOrRegister(const JS::Value& value) : constant_(true) { + MOZ_ASSERT(constant()); + new (&data.constant) JS::Value(value); + } + + MOZ_IMPLICIT ConstantOrRegister(TypedOrValueRegister reg) : constant_(false) { + MOZ_ASSERT(!constant()); + new (&data.reg) TypedOrValueRegister(reg); + } + + bool constant() const { return constant_; } + + JS::Value value() const { + MOZ_ASSERT(constant()); + return data.constant; + } + + const TypedOrValueRegister& reg() const { + MOZ_ASSERT(!constant()); + return data.reg; + } +}; + +template <typename T> +class TypedRegisterSet { + public: + using RegType = T; + using SetType = typename T::SetType; + + private: + SetType bits_; + + public: + explicit constexpr TypedRegisterSet(SetType bits) : bits_(bits) {} + + constexpr TypedRegisterSet() : bits_(0) {} + constexpr TypedRegisterSet(const TypedRegisterSet<T>& set) + : bits_(set.bits_) {} + + static inline TypedRegisterSet All() { + return TypedRegisterSet(T::Codes::AllocatableMask); + } + static inline TypedRegisterSet Intersect(const TypedRegisterSet& lhs, + const TypedRegisterSet& rhs) { + return TypedRegisterSet(lhs.bits_ & rhs.bits_); + } + static inline TypedRegisterSet Union(const TypedRegisterSet& lhs, + const TypedRegisterSet& rhs) { + return TypedRegisterSet(lhs.bits_ | rhs.bits_); + } + static inline TypedRegisterSet Not(const TypedRegisterSet& in) { + return TypedRegisterSet(~in.bits_ & T::Codes::AllocatableMask); + } + static inline TypedRegisterSet Subtract(const TypedRegisterSet& lhs, + const TypedRegisterSet& rhs) { + return TypedRegisterSet(lhs.bits_ & ~rhs.bits_); + } + static inline TypedRegisterSet VolatileNot(const TypedRegisterSet& in) { + const SetType allocatableVolatile = + T::Codes::AllocatableMask & T::Codes::VolatileMask; + return TypedRegisterSet(~in.bits_ & allocatableVolatile); + } + static inline TypedRegisterSet Volatile() { + return TypedRegisterSet(T::Codes::AllocatableMask & T::Codes::VolatileMask); + } + static inline TypedRegisterSet NonVolatile() { + return TypedRegisterSet(T::Codes::AllocatableMask & + T::Codes::NonVolatileMask); + } + + bool empty() const { return !bits_; } + void clear() { bits_ = 0; } + + bool hasRegisterIndex(T reg) const { + return !!(bits_ & (SetType(1) << reg.code())); + } + bool hasAllocatable(T reg) const { + return !(~bits_ & reg.alignedOrDominatedAliasedSet()); + } + + void addRegisterIndex(T reg) { bits_ |= (SetType(1) << reg.code()); } + void addAllocatable(T reg) { bits_ |= reg.alignedOrDominatedAliasedSet(); } + + void takeRegisterIndex(T reg) { bits_ &= ~(SetType(1) << reg.code()); } + void takeAllocatable(T reg) { bits_ &= ~reg.alignedOrDominatedAliasedSet(); } + + static constexpr RegTypeName DefaultType = RegType::DefaultType; + + template <RegTypeName Name> + SetType allLive() const { + return T::template LiveAsIndexableSet<Name>(bits_); + } + template <RegTypeName Name> + SetType allAllocatable() const { + return T::template AllocatableAsIndexableSet<Name>(bits_); + } + + static RegType FirstRegister(SetType set) { + return RegType::FromCode(RegType::FirstBit(set)); + } + static RegType LastRegister(SetType set) { + return RegType::FromCode(RegType::LastBit(set)); + } + + SetType bits() const { return bits_; } + uint32_t size() const { return T::SetSize(bits_); } + bool operator==(const TypedRegisterSet<T>& other) const { + return other.bits_ == bits_; + } + TypedRegisterSet<T> reduceSetForPush() const { + return T::ReduceSetForPush(*this); + } + uint32_t getPushSizeInBytes() const { return T::GetPushSizeInBytes(*this); } + + size_t offsetOfPushedRegister(RegType reg) const { + MOZ_ASSERT(hasRegisterIndex(reg)); + return T::OffsetOfPushedRegister(bits(), reg); + } +}; + +using GeneralRegisterSet = TypedRegisterSet<Register>; +using FloatRegisterSet = TypedRegisterSet<FloatRegister>; + +class AnyRegisterIterator; + +class RegisterSet { + GeneralRegisterSet gpr_; + FloatRegisterSet fpu_; + + friend class AnyRegisterIterator; + + public: + RegisterSet() = default; + constexpr RegisterSet(const GeneralRegisterSet& gpr, + const FloatRegisterSet& fpu) + : gpr_(gpr), fpu_(fpu) {} + static inline RegisterSet All() { + return RegisterSet(GeneralRegisterSet::All(), FloatRegisterSet::All()); + } + static inline RegisterSet Intersect(const RegisterSet& lhs, + const RegisterSet& rhs) { + return RegisterSet(GeneralRegisterSet::Intersect(lhs.gpr_, rhs.gpr_), + FloatRegisterSet::Intersect(lhs.fpu_, rhs.fpu_)); + } + static inline RegisterSet Union(const RegisterSet& lhs, + const RegisterSet& rhs) { + return RegisterSet(GeneralRegisterSet::Union(lhs.gpr_, rhs.gpr_), + FloatRegisterSet::Union(lhs.fpu_, rhs.fpu_)); + } + static inline RegisterSet Not(const RegisterSet& in) { + return RegisterSet(GeneralRegisterSet::Not(in.gpr_), + FloatRegisterSet::Not(in.fpu_)); + } + static inline RegisterSet VolatileNot(const RegisterSet& in) { + return RegisterSet(GeneralRegisterSet::VolatileNot(in.gpr_), + FloatRegisterSet::VolatileNot(in.fpu_)); + } + static inline RegisterSet Volatile() { + return RegisterSet(GeneralRegisterSet::Volatile(), + FloatRegisterSet::Volatile()); + } + + bool empty() const { return fpu_.empty() && gpr_.empty(); } + void clear() { + fpu_.clear(); + gpr_.clear(); + } + bool emptyGeneral() const { return gpr_.empty(); } + bool emptyFloat() const { return fpu_.empty(); } + + static constexpr RegTypeName DefaultType = RegTypeName::GPR; + + constexpr GeneralRegisterSet gprs() const { return gpr_; } + GeneralRegisterSet& gprs() { return gpr_; } + constexpr FloatRegisterSet fpus() const { return fpu_; } + FloatRegisterSet& fpus() { return fpu_; } + bool operator==(const RegisterSet& other) const { + return other.gpr_ == gpr_ && other.fpu_ == fpu_; + } +}; + +// [SMDOC] JIT Register-Set overview +// +// There are 2 use cases for register sets: +// +// 1. To serve as a pool of allocatable register. This is useful for working +// on the code produced by some stub where free registers are available, or +// when we can release some registers. +// +// 2. To serve as a list of typed registers. This is useful for working with +// live registers and to manipulate them with the proper instructions. This +// is used by the register allocator to fill the Safepoints. +// +// These 2 uses cases can be used on top of 3 different backend representation +// of register sets, which are either GeneralRegisterSet, FloatRegisterSet, or +// RegisterSet (for both). These classes are used to store the bit sets to +// represent each register. +// +// Each use case defines an Accessor class, such as AllocatableSetAccessor or +// LiveSetAccessor, which is parameterized with the type of the register +// set. These accessors are in charge of manipulating the register set in a +// consistent way. +// +// The RegSetCommonInterface class is used to wrap the accessors with convenient +// shortcuts which are based on the accessors. +// +// Then, to avoid to many levels of complexity while using these interfaces, +// shortcut templates are created to make it easy to distinguish between a +// register set used for allocating registers, or a register set used for making +// a collection of allocated (live) registers. +// +// This separation exists to prevent mixing LiveSet and AllocatableSet +// manipulations of the same register set, and ensure safety while avoiding +// false positive. + +template <typename RegisterSet> +class AllocatableSet; + +template <typename RegisterSet> +class LiveSet; + +// [SMDOC] JIT Register-Set (Allocatable) +// +// Base accessors classes have the minimal set of raw methods to manipulate the +// register set given as parameter in a consistent manner. These methods are: +// +// - all<Type>: Returns a bit-set of all the register of a specific type +// which are present. +// +// - has: Returns if all the bits needed to take a register are present. +// +// - takeUnchecked: Subtracts the bits used to represent the register in the +// register set. +// +// - addUnchecked: Adds the bits used to represent the register in the +// register set. + +// The AllocatableSet accessors are used to make a pool of unused +// registers. Taking or adding registers should consider the aliasing rules of +// the architecture. For example, on ARM, the following piece of code should +// work fine, knowing that the double register |d0| is composed of float +// registers |s0| and |s1|: +// +// AllocatableFloatRegisterSet regs; +// regs.add(s0); +// regs.add(s1); +// // d0 is now available. +// regs.take(d0); +// +// These accessors are useful for allocating registers within the functions used +// to generate stubs, trampolines, and inline caches (BaselineIC, IonCache). +template <typename Set> +class AllocatableSetAccessors { + public: + using RegSet = Set; + using RegType = typename RegSet::RegType; + using SetType = typename RegSet::SetType; + + protected: + RegSet set_; + + template <RegTypeName Name> + SetType all() const { + return set_.template allAllocatable<Name>(); + } + + public: + AllocatableSetAccessors() : set_() {} + explicit constexpr AllocatableSetAccessors(SetType set) : set_(set) {} + explicit constexpr AllocatableSetAccessors(RegSet set) : set_(set) {} + + bool has(RegType reg) const { return set_.hasAllocatable(reg); } + + template <RegTypeName Name> + bool hasAny(RegType reg) const { + return all<Name>() != 0; + } + + void addUnchecked(RegType reg) { set_.addAllocatable(reg); } + + void takeUnchecked(RegType reg) { set_.takeAllocatable(reg); } +}; + +// Specialization of the AllocatableSet accessors for the RegisterSet aggregate. +template <> +class AllocatableSetAccessors<RegisterSet> { + public: + using RegSet = RegisterSet; + using RegType = AnyRegister; + using SetType = char; + + protected: + RegisterSet set_; + + template <RegTypeName Name> + GeneralRegisterSet::SetType allGpr() const { + return set_.gprs().allAllocatable<Name>(); + } + template <RegTypeName Name> + FloatRegisterSet::SetType allFpu() const { + return set_.fpus().allAllocatable<Name>(); + } + + public: + AllocatableSetAccessors() : set_() {} + explicit constexpr AllocatableSetAccessors(SetType) = delete; + explicit constexpr AllocatableSetAccessors(RegisterSet set) : set_(set) {} + + bool has(Register reg) const { return set_.gprs().hasAllocatable(reg); } + bool has(FloatRegister reg) const { return set_.fpus().hasAllocatable(reg); } + + void addUnchecked(Register reg) { set_.gprs().addAllocatable(reg); } + void addUnchecked(FloatRegister reg) { set_.fpus().addAllocatable(reg); } + + void takeUnchecked(Register reg) { set_.gprs().takeAllocatable(reg); } + void takeUnchecked(FloatRegister reg) { set_.fpus().takeAllocatable(reg); } +}; + +// [SMDOC] JIT Register-Set (Live) +// +// The LiveSet accessors are used to collect a list of allocated +// registers. Taking or adding a register should *not* consider the aliases, as +// we care about interpreting the registers with the correct type. For example, +// on x64, where one float registers can be interpreted as an Simd128, a Double, +// or a Float, adding xmm0 as an Simd128, does not make the register available +// as a Double. +// +// LiveFloatRegisterSet regs; +// regs.add(xmm0.asSimd128()); +// regs.take(xmm0); // Assert! +// +// These accessors are useful for recording the result of a register allocator, +// such as what the Backtracking allocator do on the Safepoints. +template <typename Set> +class LiveSetAccessors { + public: + using RegSet = Set; + using RegType = typename RegSet::RegType; + using SetType = typename RegSet::SetType; + + protected: + RegSet set_; + + template <RegTypeName Name> + SetType all() const { + return set_.template allLive<Name>(); + } + + public: + LiveSetAccessors() : set_() {} + explicit constexpr LiveSetAccessors(SetType set) : set_(set) {} + explicit constexpr LiveSetAccessors(RegSet set) : set_(set) {} + + bool has(RegType reg) const { return set_.hasRegisterIndex(reg); } + + void addUnchecked(RegType reg) { set_.addRegisterIndex(reg); } + + void takeUnchecked(RegType reg) { set_.takeRegisterIndex(reg); } +}; + +// Specialization of the LiveSet accessors for the RegisterSet aggregate. +template <> +class LiveSetAccessors<RegisterSet> { + public: + using RegSet = RegisterSet; + using RegType = AnyRegister; + using SetType = char; + + protected: + RegisterSet set_; + + template <RegTypeName Name> + GeneralRegisterSet::SetType allGpr() const { + return set_.gprs().allLive<Name>(); + } + template <RegTypeName Name> + FloatRegisterSet::SetType allFpu() const { + return set_.fpus().allLive<Name>(); + } + + public: + LiveSetAccessors() : set_() {} + explicit constexpr LiveSetAccessors(SetType) = delete; + explicit constexpr LiveSetAccessors(RegisterSet set) : set_(set) {} + + bool has(Register reg) const { return set_.gprs().hasRegisterIndex(reg); } + bool has(FloatRegister reg) const { + return set_.fpus().hasRegisterIndex(reg); + } + + void addUnchecked(Register reg) { set_.gprs().addRegisterIndex(reg); } + void addUnchecked(FloatRegister reg) { set_.fpus().addRegisterIndex(reg); } + + void takeUnchecked(Register reg) { set_.gprs().takeRegisterIndex(reg); } + void takeUnchecked(FloatRegister reg) { set_.fpus().takeRegisterIndex(reg); } +}; + +#define DEFINE_ACCESSOR_CONSTRUCTORS_(REGSET) \ + typedef typename Parent::RegSet RegSet; \ + typedef typename Parent::RegType RegType; \ + typedef typename Parent::SetType SetType; \ + \ + constexpr REGSET() : Parent() {} \ + explicit constexpr REGSET(SetType set) : Parent(set) {} \ + explicit constexpr REGSET(RegSet set) : Parent(set) {} + +// This class adds checked accessors on top of the unchecked variants defined by +// AllocatableSet and LiveSet accessors. Also it defines interface which are +// specialized to the register set implementation, such as |getAny| and +// |takeAny| variants. +template <class Accessors, typename Set> +class SpecializedRegSet : public Accessors { + using Parent = Accessors; + + public: + DEFINE_ACCESSOR_CONSTRUCTORS_(SpecializedRegSet) + + SetType bits() const { return this->Parent::set_.bits(); } + + using Parent::has; + + using Parent::addUnchecked; + void add(RegType reg) { + MOZ_ASSERT(!this->has(reg)); + addUnchecked(reg); + } + + using Parent::takeUnchecked; + void take(RegType reg) { + MOZ_ASSERT(this->has(reg)); + takeUnchecked(reg); + } + + template <RegTypeName Name> + bool hasAny() const { + return Parent::template all<Name>() != 0; + } + + template <RegTypeName Name = RegSet::DefaultType> + RegType getFirst() const { + SetType set = Parent::template all<Name>(); + MOZ_ASSERT(set); + return RegSet::FirstRegister(set); + } + template <RegTypeName Name = RegSet::DefaultType> + RegType getLast() const { + SetType set = Parent::template all<Name>(); + MOZ_ASSERT(set); + return RegSet::LastRegister(set); + } + template <RegTypeName Name = RegSet::DefaultType> + RegType getAny() const { + // The choice of first or last here is mostly arbitrary, as they are + // about the same speed on popular architectures. We choose first, as + // it has the advantage of using the "lower" registers more often. These + // registers are sometimes more efficient (e.g. optimized encodings for + // EAX on x86). + return getFirst<Name>(); + } + + template <RegTypeName Name = RegSet::DefaultType> + RegType getAnyExcluding(RegType preclude) { + if (!this->has(preclude)) { + return getAny<Name>(); + } + + take(preclude); + RegType result = getAny<Name>(); + add(preclude); + return result; + } + + template <RegTypeName Name = RegSet::DefaultType> + RegType takeAny() { + RegType reg = getAny<Name>(); + take(reg); + return reg; + } + template <RegTypeName Name = RegSet::DefaultType> + RegType takeFirst() { + RegType reg = getFirst<Name>(); + take(reg); + return reg; + } + template <RegTypeName Name = RegSet::DefaultType> + RegType takeLast() { + RegType reg = getLast<Name>(); + take(reg); + return reg; + } + + ValueOperand takeAnyValue() { +#if defined(JS_NUNBOX32) + return ValueOperand(takeAny<RegTypeName::GPR>(), + takeAny<RegTypeName::GPR>()); +#elif defined(JS_PUNBOX64) + return ValueOperand(takeAny<RegTypeName::GPR>()); +#else +# error "Bad architecture" +#endif + } + + bool aliases(ValueOperand v) const { +#ifdef JS_NUNBOX32 + return this->has(v.typeReg()) || this->has(v.payloadReg()); +#else + return this->has(v.valueReg()); +#endif + } + + template <RegTypeName Name = RegSet::DefaultType> + RegType takeAnyExcluding(RegType preclude) { + RegType reg = getAnyExcluding<Name>(preclude); + take(reg); + return reg; + } +}; + +// Specialization of the accessors for the RegisterSet aggregate. +template <class Accessors> +class SpecializedRegSet<Accessors, RegisterSet> : public Accessors { + using Parent = Accessors; + + public: + DEFINE_ACCESSOR_CONSTRUCTORS_(SpecializedRegSet) + + GeneralRegisterSet gprs() const { return this->Parent::set_.gprs(); } + GeneralRegisterSet& gprs() { return this->Parent::set_.gprs(); } + FloatRegisterSet fpus() const { return this->Parent::set_.fpus(); } + FloatRegisterSet& fpus() { return this->Parent::set_.fpus(); } + + bool emptyGeneral() const { return this->Parent::set_.emptyGeneral(); } + bool emptyFloat() const { return this->Parent::set_.emptyFloat(); } + + using Parent::has; + bool has(AnyRegister reg) const { + return reg.isFloat() ? this->has(reg.fpu()) : this->has(reg.gpr()); + } + + template <RegTypeName Name> + bool hasAny() const { + if (Name == RegTypeName::GPR) { + return Parent::template allGpr<RegTypeName::GPR>() != 0; + } + return Parent::template allFpu<Name>() != 0; + } + + using Parent::addUnchecked; + void addUnchecked(AnyRegister reg) { + if (reg.isFloat()) { + addUnchecked(reg.fpu()); + } else { + addUnchecked(reg.gpr()); + } + } + + void add(Register reg) { + MOZ_ASSERT(!this->has(reg)); + addUnchecked(reg); + } + void add(FloatRegister reg) { + MOZ_ASSERT(!this->has(reg)); + addUnchecked(reg); + } + void add(AnyRegister reg) { + if (reg.isFloat()) { + add(reg.fpu()); + } else { + add(reg.gpr()); + } + } + + using Parent::takeUnchecked; + void takeUnchecked(AnyRegister reg) { + if (reg.isFloat()) { + takeUnchecked(reg.fpu()); + } else { + takeUnchecked(reg.gpr()); + } + } + + void take(Register reg) { +#ifdef DEBUG + bool hasReg = this->has(reg); + MOZ_ASSERT(hasReg); +#endif + takeUnchecked(reg); + } + void take(FloatRegister reg) { + MOZ_ASSERT(this->has(reg)); + takeUnchecked(reg); + } + void take(AnyRegister reg) { + if (reg.isFloat()) { + take(reg.fpu()); + } else { + take(reg.gpr()); + } + } + + Register getAnyGeneral() const { + GeneralRegisterSet::SetType set = + Parent::template allGpr<RegTypeName::GPR>(); + MOZ_ASSERT(set); + return GeneralRegisterSet::FirstRegister(set); + } + template <RegTypeName Name = RegTypeName::Float64> + FloatRegister getAnyFloat() const { + FloatRegisterSet::SetType set = Parent::template allFpu<Name>(); + MOZ_ASSERT(set); + return FloatRegisterSet::FirstRegister(set); + } + + Register takeAnyGeneral() { + Register reg = getAnyGeneral(); + take(reg); + return reg; + } + template <RegTypeName Name = RegTypeName::Float64> + FloatRegister takeAnyFloat() { + FloatRegister reg = getAnyFloat<Name>(); + take(reg); + return reg; + } + ValueOperand takeAnyValue() { +#if defined(JS_NUNBOX32) + return ValueOperand(takeAnyGeneral(), takeAnyGeneral()); +#elif defined(JS_PUNBOX64) + return ValueOperand(takeAnyGeneral()); +#else +# error "Bad architecture" +#endif + } +}; + +// Interface which is common to all register set implementations. It overloads +// |add|, |take| and |takeUnchecked| methods for types such as |ValueOperand|, +// |TypedOrValueRegister|, and |Register64|. +template <class Accessors, typename Set> +class CommonRegSet : public SpecializedRegSet<Accessors, Set> { + typedef SpecializedRegSet<Accessors, Set> Parent; + + public: + DEFINE_ACCESSOR_CONSTRUCTORS_(CommonRegSet) + + RegSet set() const { return this->Parent::set_; } + RegSet& set() { return this->Parent::set_; } + + bool empty() const { return this->Parent::set_.empty(); } + void clear() { this->Parent::set_.clear(); } + + using Parent::add; + void add(ValueOperand value) { +#if defined(JS_NUNBOX32) + add(value.payloadReg()); + add(value.typeReg()); +#elif defined(JS_PUNBOX64) + add(value.valueReg()); +#else +# error "Bad architecture" +#endif + } + void add(Register64 reg) { +#if JS_BITS_PER_WORD == 32 + add(reg.high); + add(reg.low); +#else + add(reg.reg); +#endif + } + + using Parent::addUnchecked; + void addUnchecked(ValueOperand value) { +#if defined(JS_NUNBOX32) + addUnchecked(value.payloadReg()); + addUnchecked(value.typeReg()); +#elif defined(JS_PUNBOX64) + addUnchecked(value.valueReg()); +#else +# error "Bad architecture" +#endif + } + void addUnchecked(Register64 reg) { +#if JS_BITS_PER_WORD == 32 + take(reg.high); + take(reg.low); +#else + take(reg.reg); +#endif + } + + void add(TypedOrValueRegister reg) { + if (reg.hasValue()) { + add(reg.valueReg()); + } else if (reg.hasTyped()) { + add(reg.typedReg()); + } + } + + using Parent::take; + void take(ValueOperand value) { +#if defined(JS_NUNBOX32) + take(value.payloadReg()); + take(value.typeReg()); +#elif defined(JS_PUNBOX64) + take(value.valueReg()); +#else +# error "Bad architecture" +#endif + } + void take(TypedOrValueRegister reg) { + if (reg.hasValue()) { + take(reg.valueReg()); + } else if (reg.hasTyped()) { + take(reg.typedReg()); + } + } + void take(Register64 reg) { +#if JS_BITS_PER_WORD == 32 + take(reg.high); + take(reg.low); +#else + take(reg.reg); +#endif + } + + using Parent::takeUnchecked; + void takeUnchecked(ValueOperand value) { +#if defined(JS_NUNBOX32) + takeUnchecked(value.payloadReg()); + takeUnchecked(value.typeReg()); +#elif defined(JS_PUNBOX64) + takeUnchecked(value.valueReg()); +#else +# error "Bad architecture" +#endif + } + void takeUnchecked(TypedOrValueRegister reg) { + if (reg.hasValue()) { + takeUnchecked(reg.valueReg()); + } else if (reg.hasTyped()) { + takeUnchecked(reg.typedReg()); + } + } + void takeUnchecked(Register64 reg) { +#if JS_BITS_PER_WORD == 32 + takeUnchecked(reg.high); + takeUnchecked(reg.low); +#else + takeUnchecked(reg.reg); +#endif + } +}; + +// These classes do not provide any additional members, they only use their +// constructors to forward to the common interface for all register sets. The +// only benefit of these classes is to provide user friendly names. +template <typename Set> +class LiveSet : public CommonRegSet<LiveSetAccessors<Set>, Set> { + typedef CommonRegSet<LiveSetAccessors<Set>, Set> Parent; + + public: + DEFINE_ACCESSOR_CONSTRUCTORS_(LiveSet) +}; + +template <typename Set> +class AllocatableSet : public CommonRegSet<AllocatableSetAccessors<Set>, Set> { + typedef CommonRegSet<AllocatableSetAccessors<Set>, Set> Parent; + + public: + DEFINE_ACCESSOR_CONSTRUCTORS_(AllocatableSet) + + LiveSet<Set> asLiveSet() const { return LiveSet<Set>(this->set()); } +}; + +#define DEFINE_ACCESSOR_CONSTRUCTORS_FOR_REGISTERSET_(REGSET) \ + typedef Parent::RegSet RegSet; \ + typedef Parent::RegType RegType; \ + typedef Parent::SetType SetType; \ + \ + constexpr REGSET() : Parent() {} \ + explicit constexpr REGSET(SetType) = delete; \ + explicit constexpr REGSET(RegSet set) : Parent(set) {} \ + constexpr REGSET(GeneralRegisterSet gpr, FloatRegisterSet fpu) \ + : Parent(RegisterSet(gpr, fpu)) {} \ + REGSET(REGSET<GeneralRegisterSet> gpr, REGSET<FloatRegisterSet> fpu) \ + : Parent(RegisterSet(gpr.set(), fpu.set())) {} + +template <> +class LiveSet<RegisterSet> + : public CommonRegSet<LiveSetAccessors<RegisterSet>, RegisterSet> { + // Note: We have to provide a qualified name for LiveSetAccessors, as it is + // interpreted as being the specialized class name inherited from the parent + // class specialization. + typedef CommonRegSet<jit::LiveSetAccessors<RegisterSet>, RegisterSet> Parent; + + public: + DEFINE_ACCESSOR_CONSTRUCTORS_FOR_REGISTERSET_(LiveSet) +}; + +template <> +class AllocatableSet<RegisterSet> + : public CommonRegSet<AllocatableSetAccessors<RegisterSet>, RegisterSet> { + // Note: We have to provide a qualified name for AllocatableSetAccessors, as + // it is interpreted as being the specialized class name inherited from the + // parent class specialization. + typedef CommonRegSet<jit::AllocatableSetAccessors<RegisterSet>, RegisterSet> + Parent; + + public: + DEFINE_ACCESSOR_CONSTRUCTORS_FOR_REGISTERSET_(AllocatableSet) + + LiveSet<RegisterSet> asLiveSet() const { + return LiveSet<RegisterSet>(this->set()); + } +}; + +#undef DEFINE_ACCESSOR_CONSTRUCTORS_FOR_REGISTERSET_ +#undef DEFINE_ACCESSOR_CONSTRUCTORS_ + +using AllocatableGeneralRegisterSet = AllocatableSet<GeneralRegisterSet>; +using AllocatableFloatRegisterSet = AllocatableSet<FloatRegisterSet>; +using AllocatableRegisterSet = AllocatableSet<RegisterSet>; + +using LiveGeneralRegisterSet = LiveSet<GeneralRegisterSet>; +using LiveFloatRegisterSet = LiveSet<FloatRegisterSet>; +using LiveRegisterSet = LiveSet<RegisterSet>; + +// iterates in whatever order happens to be convenient. +// Use TypedRegisterBackwardIterator or TypedRegisterForwardIterator if a +// specific order is required. +template <typename T> +class TypedRegisterIterator { + LiveSet<TypedRegisterSet<T>> regset_; + + public: + explicit TypedRegisterIterator(TypedRegisterSet<T> regset) + : regset_(regset) {} + explicit TypedRegisterIterator(LiveSet<TypedRegisterSet<T>> regset) + : regset_(regset) {} + TypedRegisterIterator(const TypedRegisterIterator& other) + : regset_(other.regset_) {} + + bool more() const { return !regset_.empty(); } + TypedRegisterIterator<T>& operator++() { + regset_.template takeAny<RegTypeName::Any>(); + return *this; + } + T operator*() const { return regset_.template getAny<RegTypeName::Any>(); } +}; + +// iterates backwards, that is, rn to r0 +template <typename T> +class TypedRegisterBackwardIterator { + LiveSet<TypedRegisterSet<T>> regset_; + + public: + explicit TypedRegisterBackwardIterator(TypedRegisterSet<T> regset) + : regset_(regset) {} + explicit TypedRegisterBackwardIterator(LiveSet<TypedRegisterSet<T>> regset) + : regset_(regset) {} + TypedRegisterBackwardIterator(const TypedRegisterBackwardIterator& other) + : regset_(other.regset_) {} + + bool more() const { return !regset_.empty(); } + TypedRegisterBackwardIterator<T>& operator++() { + regset_.template takeLast<RegTypeName::Any>(); + return *this; + } + T operator*() const { return regset_.template getLast<RegTypeName::Any>(); } +}; + +// iterates forwards, that is r0 to rn +template <typename T> +class TypedRegisterForwardIterator { + LiveSet<TypedRegisterSet<T>> regset_; + + public: + explicit TypedRegisterForwardIterator(TypedRegisterSet<T> regset) + : regset_(regset) {} + explicit TypedRegisterForwardIterator(LiveSet<TypedRegisterSet<T>> regset) + : regset_(regset) {} + TypedRegisterForwardIterator(const TypedRegisterForwardIterator& other) + : regset_(other.regset_) {} + + bool more() const { return !regset_.empty(); } + TypedRegisterForwardIterator<T>& operator++() { + regset_.template takeFirst<RegTypeName::Any>(); + return *this; + } + T operator*() const { return regset_.template getFirst<RegTypeName::Any>(); } +}; + +using GeneralRegisterIterator = TypedRegisterIterator<Register>; +using FloatRegisterIterator = TypedRegisterIterator<FloatRegister>; +using GeneralRegisterBackwardIterator = TypedRegisterBackwardIterator<Register>; +using FloatRegisterBackwardIterator = + TypedRegisterBackwardIterator<FloatRegister>; +using GeneralRegisterForwardIterator = TypedRegisterForwardIterator<Register>; +using FloatRegisterForwardIterator = + TypedRegisterForwardIterator<FloatRegister>; + +class AnyRegisterIterator { + GeneralRegisterIterator geniter_; + FloatRegisterIterator floatiter_; + + public: + AnyRegisterIterator() + : geniter_(GeneralRegisterSet::All()), + floatiter_(FloatRegisterSet::All()) {} + AnyRegisterIterator(GeneralRegisterSet genset, FloatRegisterSet floatset) + : geniter_(genset), floatiter_(floatset) {} + explicit AnyRegisterIterator(const RegisterSet& set) + : geniter_(set.gpr_), floatiter_(set.fpu_) {} + explicit AnyRegisterIterator(const LiveSet<RegisterSet>& set) + : geniter_(set.gprs()), floatiter_(set.fpus()) {} + AnyRegisterIterator(const AnyRegisterIterator& other) = default; + bool more() const { return geniter_.more() || floatiter_.more(); } + AnyRegisterIterator& operator++() { + if (geniter_.more()) { + ++geniter_; + } else { + ++floatiter_; + } + return *this; + } + AnyRegister operator*() const { + if (geniter_.more()) { + return AnyRegister(*geniter_); + } + return AnyRegister(*floatiter_); + } +}; + +class ABIArg { + public: + enum Kind { + GPR, +#ifdef JS_CODEGEN_REGISTER_PAIR + GPR_PAIR, +#endif + FPU, + Stack, + Uninitialized = -1 + }; + + private: + Kind kind_; + union { + Register::Code gpr_; + FloatRegister::Code fpu_; + uint32_t offset_; + } u; + + public: + ABIArg() : kind_(Uninitialized) { u.offset_ = -1; } + explicit ABIArg(Register gpr) : kind_(GPR) { u.gpr_ = gpr.code(); } + explicit ABIArg(Register gprLow, Register gprHigh) { +#if defined(JS_CODEGEN_REGISTER_PAIR) + kind_ = GPR_PAIR; +#else + MOZ_CRASH("Unsupported type of ABI argument."); +#endif + u.gpr_ = gprLow.code(); + MOZ_ASSERT(u.gpr_ % 2 == 0); + MOZ_ASSERT(u.gpr_ + 1 == gprHigh.code()); + } + explicit ABIArg(FloatRegister fpu) : kind_(FPU) { u.fpu_ = fpu.code(); } + explicit ABIArg(uint32_t offset) : kind_(Stack) { u.offset_ = offset; } + + Kind kind() const { + MOZ_ASSERT(kind_ != Uninitialized); + return kind_; + } +#ifdef JS_CODEGEN_REGISTER_PAIR + bool isGeneralRegPair() const { return kind() == GPR_PAIR; } +#else + bool isGeneralRegPair() const { return false; } +#endif + + Register gpr() const { + MOZ_ASSERT(kind() == GPR); + return Register::FromCode(u.gpr_); + } + Register64 gpr64() const { +#ifdef JS_PUNBOX64 + return Register64(gpr()); +#else + return Register64(oddGpr(), evenGpr()); +#endif + } + Register evenGpr() const { + MOZ_ASSERT(isGeneralRegPair()); + return Register::FromCode(u.gpr_); + } + Register oddGpr() const { + MOZ_ASSERT(isGeneralRegPair()); + return Register::FromCode(u.gpr_ + 1); + } + FloatRegister fpu() const { + MOZ_ASSERT(kind() == FPU); + return FloatRegister::FromCode(u.fpu_); + } + uint32_t offsetFromArgBase() const { + MOZ_ASSERT(kind() == Stack); + return u.offset_; + } + + bool argInRegister() const { return kind() != Stack; } + AnyRegister reg() const { + return kind() == GPR ? AnyRegister(gpr()) : AnyRegister(fpu()); + } + + bool operator==(const ABIArg& rhs) const { + if (kind_ != rhs.kind_) { + return false; + } + + switch (kind_) { + case GPR: + return u.gpr_ == rhs.u.gpr_; +#if defined(JS_CODEGEN_REGISTER_PAIR) + case GPR_PAIR: + return u.gpr_ == rhs.u.gpr_; +#endif + case FPU: + return u.fpu_ == rhs.u.fpu_; + case Stack: + return u.offset_ == rhs.u.offset_; + case Uninitialized: + return true; + } + MOZ_CRASH("Invalid value for ABIArg kind"); + } + + bool operator!=(const ABIArg& rhs) const { return !(*this == rhs); } +}; + +// Get the set of registers which should be saved by a block of code which +// clobbers all registers besides |unused|, but does not clobber floating point +// registers. +inline LiveGeneralRegisterSet SavedNonVolatileRegisters( + const AllocatableGeneralRegisterSet& unused) { + LiveGeneralRegisterSet result; + + for (GeneralRegisterIterator iter(GeneralRegisterSet::NonVolatile()); + iter.more(); ++iter) { + Register reg = *iter; + if (!unused.has(reg)) { + result.add(reg); + } + } + + // Some platforms require the link register to be saved, if calls can be made. +#if defined(JS_CODEGEN_ARM) + result.add(Register::FromCode(Registers::lr)); +#elif defined(JS_CODEGEN_ARM64) + result.add(Register::FromCode(Registers::lr)); +#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \ + defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_RISCV64) + result.add(Register::FromCode(Registers::ra)); +#endif + + return result; +} + +} // namespace jit +} // namespace js + +#endif /* jit_RegisterSets_h */ |