From 6bf0a5cb5034a7e684dcc3500e841785237ce2dd Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 19:32:43 +0200 Subject: Adding upstream version 1:115.7.0. Signed-off-by: Daniel Baumann --- js/src/jit/x86-shared/Assembler-x86-shared.cpp | 355 +++++++++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 js/src/jit/x86-shared/Assembler-x86-shared.cpp (limited to 'js/src/jit/x86-shared/Assembler-x86-shared.cpp') diff --git a/js/src/jit/x86-shared/Assembler-x86-shared.cpp b/js/src/jit/x86-shared/Assembler-x86-shared.cpp new file mode 100644 index 0000000000..65fd124cf8 --- /dev/null +++ b/js/src/jit/x86-shared/Assembler-x86-shared.cpp @@ -0,0 +1,355 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Maybe.h" + +#include + +#include "jit/AutoWritableJitCode.h" +#if defined(JS_CODEGEN_X86) +# include "jit/x86/MacroAssembler-x86.h" +#elif defined(JS_CODEGEN_X64) +# include "jit/x64/MacroAssembler-x64.h" +#else +# error "Wrong architecture. Only x86 and x64 should build this file!" +#endif + +#ifdef _MSC_VER +# include // for __cpuid +# if defined(_M_X64) && (_MSC_FULL_VER >= 160040219) +# include // for _xgetbv +# endif +#endif + +using namespace js; +using namespace js::jit; + +void AssemblerX86Shared::copyJumpRelocationTable(uint8_t* dest) { + if (jumpRelocations_.length()) { + memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length()); + } +} + +void AssemblerX86Shared::copyDataRelocationTable(uint8_t* dest) { + if (dataRelocations_.length()) { + memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length()); + } +} + +/* static */ +void AssemblerX86Shared::TraceDataRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + mozilla::Maybe awjc; + + while (reader.more()) { + size_t offset = reader.readUnsigned(); + MOZ_ASSERT(offset >= sizeof(void*) && offset <= code->instructionsSize()); + + uint8_t* src = code->raw() + offset; + void* data = X86Encoding::GetPointer(src); + +#ifdef JS_PUNBOX64 + // Data relocations can be for Values or for raw pointers. If a Value is + // zero-tagged, we can trace it as if it were a raw pointer. If a Value + // is not zero-tagged, we have to interpret it as a Value to ensure that the + // tag bits are masked off to recover the actual pointer. + + uintptr_t word = reinterpret_cast(data); + if (word >> JSVAL_TAG_SHIFT) { + // This relocation is a Value with a non-zero tag. + Value value = Value::fromRawBits(word); + MOZ_ASSERT_IF(value.isGCThing(), + gc::IsCellPointerValid(value.toGCThing())); + TraceManuallyBarrieredEdge(trc, &value, "jit-masm-value"); + if (word != value.asRawBits()) { + if (awjc.isNothing()) { + awjc.emplace(code); + } + X86Encoding::SetPointer(src, value.bitsAsPunboxPointer()); + } + continue; + } +#endif + + // This relocation is a raw pointer or a Value with a zero tag. + gc::Cell* cell = static_cast(data); + MOZ_ASSERT(gc::IsCellPointerValid(cell)); + TraceManuallyBarrieredGenericPointerEdge(trc, &cell, "jit-masm-ptr"); + if (cell != data) { + if (awjc.isNothing()) { + awjc.emplace(code); + } + X86Encoding::SetPointer(src, cell); + } + } +} + +void AssemblerX86Shared::executableCopy(void* buffer) { + masm.executableCopy(buffer); +} + +void AssemblerX86Shared::processCodeLabels(uint8_t* rawCode) { + for (const CodeLabel& label : codeLabels_) { + Bind(rawCode, label); + } +} + +AssemblerX86Shared::Condition AssemblerX86Shared::InvertCondition( + Condition cond) { + switch (cond) { + case Zero: + return NonZero; + case NonZero: + return Zero; + case LessThan: + return GreaterThanOrEqual; + case LessThanOrEqual: + return GreaterThan; + case GreaterThan: + return LessThanOrEqual; + case GreaterThanOrEqual: + return LessThan; + case Above: + return BelowOrEqual; + case AboveOrEqual: + return Below; + case Below: + return AboveOrEqual; + case BelowOrEqual: + return Above; + default: + MOZ_CRASH("unexpected condition"); + } +} + +AssemblerX86Shared::Condition AssemblerX86Shared::UnsignedCondition( + Condition cond) { + switch (cond) { + case Zero: + case NonZero: + return cond; + case LessThan: + case Below: + return Below; + case LessThanOrEqual: + case BelowOrEqual: + return BelowOrEqual; + case GreaterThan: + case Above: + return Above; + case AboveOrEqual: + case GreaterThanOrEqual: + return AboveOrEqual; + default: + MOZ_CRASH("unexpected condition"); + } +} + +AssemblerX86Shared::Condition AssemblerX86Shared::ConditionWithoutEqual( + Condition cond) { + switch (cond) { + case LessThan: + case LessThanOrEqual: + return LessThan; + case Below: + case BelowOrEqual: + return Below; + case GreaterThan: + case GreaterThanOrEqual: + return GreaterThan; + case Above: + case AboveOrEqual: + return Above; + default: + MOZ_CRASH("unexpected condition"); + } +} + +AssemblerX86Shared::DoubleCondition AssemblerX86Shared::InvertCondition( + DoubleCondition cond) { + switch (cond) { + case DoubleEqual: + return DoubleNotEqualOrUnordered; + case DoubleEqualOrUnordered: + return DoubleNotEqual; + case DoubleNotEqualOrUnordered: + return DoubleEqual; + case DoubleNotEqual: + return DoubleEqualOrUnordered; + case DoubleLessThan: + return DoubleGreaterThanOrEqualOrUnordered; + case DoubleLessThanOrUnordered: + return DoubleGreaterThanOrEqual; + case DoubleLessThanOrEqual: + return DoubleGreaterThanOrUnordered; + case DoubleLessThanOrEqualOrUnordered: + return DoubleGreaterThan; + case DoubleGreaterThan: + return DoubleLessThanOrEqualOrUnordered; + case DoubleGreaterThanOrUnordered: + return DoubleLessThanOrEqual; + case DoubleGreaterThanOrEqual: + return DoubleLessThanOrUnordered; + case DoubleGreaterThanOrEqualOrUnordered: + return DoubleLessThan; + default: + MOZ_CRASH("unexpected condition"); + } +} + +CPUInfo::SSEVersion CPUInfo::maxSSEVersion = UnknownSSE; +CPUInfo::SSEVersion CPUInfo::maxEnabledSSEVersion = UnknownSSE; +bool CPUInfo::avxPresent = false; +#ifdef ENABLE_WASM_AVX +bool CPUInfo::avxEnabled = true; +#else +bool CPUInfo::avxEnabled = false; +#endif +bool CPUInfo::popcntPresent = false; +bool CPUInfo::bmi1Present = false; +bool CPUInfo::bmi2Present = false; +bool CPUInfo::lzcntPresent = false; +bool CPUInfo::avx2Present = false; +bool CPUInfo::fmaPresent = false; + +namespace js { +namespace jit { +bool CPUFlagsHaveBeenComputed() { return CPUInfo::FlagsHaveBeenComputed(); } +} // namespace jit +} // namespace js + +static uintptr_t ReadXGETBV() { + // We use a variety of low-level mechanisms to get at the xgetbv + // instruction, including spelling out the xgetbv instruction as bytes, + // because older compilers and assemblers may not recognize the instruction + // by name. + size_t xcr0EAX = 0; +#if defined(_XCR_XFEATURE_ENABLED_MASK) + xcr0EAX = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); +#elif defined(__GNUC__) + // xgetbv returns its results in %eax and %edx, and for our purposes here, + // we're only interested in the %eax value. + asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0EAX) : "c"(0) : "%edx"); +#elif defined(_MSC_VER) && defined(_M_IX86) + __asm { + xor ecx, ecx + _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0 + mov xcr0EAX, eax + } +#endif + return xcr0EAX; +} + +static void ReadCPUInfo(int* flagsEax, int* flagsEbx, int* flagsEcx, + int* flagsEdx) { +#ifdef _MSC_VER + int cpuinfo[4]; + __cpuid(cpuinfo, *flagsEax); + *flagsEax = cpuinfo[0]; + *flagsEbx = cpuinfo[1]; + *flagsEcx = cpuinfo[2]; + *flagsEdx = cpuinfo[3]; +#elif defined(__GNUC__) + // Some older 32-bits processors don't fill the ecx register with cpuid, so + // clobber it before calling cpuid, so that there's no risk of picking + // random bits indicating SSE3/SSE4 are present. Also make sure that it's + // set to 0 as an input for BMI detection on all platforms. + *flagsEcx = 0; +# ifdef JS_CODEGEN_X64 + asm("cpuid;" + : "+a"(*flagsEax), "=b"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx)); +# else + // On x86, preserve ebx. The compiler needs it for PIC mode. + asm("mov %%ebx, %%edi;" + "cpuid;" + "xchg %%edi, %%ebx;" + : "+a"(*flagsEax), "=D"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx)); +# endif +#else +# error "Unsupported compiler" +#endif +} + +void CPUInfo::ComputeFlags() { + MOZ_ASSERT(!FlagsHaveBeenComputed()); + + int flagsEax = 1; + int flagsEbx = 0; + int flagsEcx = 0; + int flagsEdx = 0; + ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx); + + static constexpr int SSEBit = 1 << 25; + static constexpr int SSE2Bit = 1 << 26; + static constexpr int SSE3Bit = 1 << 0; + static constexpr int SSSE3Bit = 1 << 9; + static constexpr int SSE41Bit = 1 << 19; + static constexpr int SSE42Bit = 1 << 20; + + if (flagsEcx & SSE42Bit) { + maxSSEVersion = SSE4_2; + } else if (flagsEcx & SSE41Bit) { + maxSSEVersion = SSE4_1; + } else if (flagsEcx & SSSE3Bit) { + maxSSEVersion = SSSE3; + } else if (flagsEcx & SSE3Bit) { + maxSSEVersion = SSE3; + } else if (flagsEdx & SSE2Bit) { + maxSSEVersion = SSE2; + } else if (flagsEdx & SSEBit) { + maxSSEVersion = SSE; + } else { + maxSSEVersion = NoSSE; + } + + if (maxEnabledSSEVersion != UnknownSSE) { + maxSSEVersion = std::min(maxSSEVersion, maxEnabledSSEVersion); + } + + static constexpr int AVXBit = 1 << 28; + static constexpr int XSAVEBit = 1 << 27; + avxPresent = (flagsEcx & AVXBit) && (flagsEcx & XSAVEBit) && avxEnabled; + + // If the hardware supports AVX, check whether the OS supports it too. + if (avxPresent) { + size_t xcr0EAX = ReadXGETBV(); + static constexpr int xcr0SSEBit = 1 << 1; + static constexpr int xcr0AVXBit = 1 << 2; + avxPresent = (xcr0EAX & xcr0SSEBit) && (xcr0EAX & xcr0AVXBit); + } + + // CMOV instruction are supposed to be supported by all CPU which have SSE2 + // enabled. While this might be true, this is not guaranteed by any + // documentation, nor AMD, nor Intel. + static constexpr int CMOVBit = 1 << 15; + MOZ_RELEASE_ASSERT(flagsEdx & CMOVBit, + "CMOVcc instruction is not recognized by this CPU."); + + static constexpr int POPCNTBit = 1 << 23; + popcntPresent = (flagsEcx & POPCNTBit); + + // Use the avxEnabled flag to enable/disable FMA. + static constexpr int FMABit = 1 << 12; + fmaPresent = (flagsEcx & FMABit) && avxEnabled; + + flagsEax = 0x80000001; + ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx); + + static constexpr int LZCNTBit = 1 << 5; + lzcntPresent = (flagsEcx & LZCNTBit); + + flagsEax = 0x7; + ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx); + + static constexpr int BMI1Bit = 1 << 3; + static constexpr int BMI2Bit = 1 << 8; + static constexpr int AVX2Bit = 1 << 5; + bmi1Present = (flagsEbx & BMI1Bit); + bmi2Present = bmi1Present && (flagsEbx & BMI2Bit); + avx2Present = avxPresent && (flagsEbx & AVX2Bit); + + MOZ_ASSERT(FlagsHaveBeenComputed()); +} -- cgit v1.2.3