path: root/js/src/jit/x86-shared/Assembler-x86-shared.cpp

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
 * vim: set ts=8 sts=2 et sw=2 tw=80:
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "mozilla/Maybe.h"

#include <algorithm>

#include "jit/AutoWritableJitCode.h"
#if defined(JS_CODEGEN_X86)
#  include "jit/x86/MacroAssembler-x86.h"
#elif defined(JS_CODEGEN_X64)
#  include "jit/x64/MacroAssembler-x64.h"
#else
#  error "Wrong architecture. Only x86 and x64 should build this file!"
#endif

#ifdef _MSC_VER
#  include <intrin.h>  // for __cpuid
#  if defined(_M_X64) && (_MSC_FULL_VER >= 160040219)
#    include <immintrin.h>  // for _xgetbv
#  endif
#endif

using namespace js;
using namespace js::jit;

void AssemblerX86Shared::copyJumpRelocationTable(uint8_t* dest) {
  if (jumpRelocations_.length()) {
    memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
  }
}

void AssemblerX86Shared::copyDataRelocationTable(uint8_t* dest) {
  if (dataRelocations_.length()) {
    memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
  }
}

/* static */
void AssemblerX86Shared::TraceDataRelocations(JSTracer* trc, JitCode* code,
                                              CompactBufferReader& reader) {
  mozilla::Maybe<AutoWritableJitCode> awjc;

  while (reader.more()) {
    size_t offset = reader.readUnsigned();
    MOZ_ASSERT(offset >= sizeof(void*) && offset <= code->instructionsSize());

    uint8_t* src = code->raw() + offset;
    void* data = X86Encoding::GetPointer(src);

#ifdef JS_PUNBOX64
    // Data relocations can be for Values or for raw pointers. If a Value is
    // zero-tagged, we can trace it as if it were a raw pointer. If a Value
    // is not zero-tagged, we have to interpret it as a Value to ensure that the
    // tag bits are masked off to recover the actual pointer.

    uintptr_t word = reinterpret_cast<uintptr_t>(data);
    if (word >> JSVAL_TAG_SHIFT) {
      // This relocation is a Value with a non-zero tag.
      Value value = Value::fromRawBits(word);
      MOZ_ASSERT_IF(value.isGCThing(),
                    gc::IsCellPointerValid(value.toGCThing()));
      TraceManuallyBarrieredEdge(trc, &value, "jit-masm-value");
      if (word != value.asRawBits()) {
        if (awjc.isNothing()) {
          awjc.emplace(code);
        }
        X86Encoding::SetPointer(src, value.bitsAsPunboxPointer());
      }
      continue;
    }
#endif

    // This relocation is a raw pointer or a Value with a zero tag.
    gc::Cell* cell = static_cast<gc::Cell*>(data);
    MOZ_ASSERT(gc::IsCellPointerValid(cell));
    TraceManuallyBarrieredGenericPointerEdge(trc, &cell, "jit-masm-ptr");
    if (cell != data) {
      if (awjc.isNothing()) {
        awjc.emplace(code);
      }
      X86Encoding::SetPointer(src, cell);
    }
  }
}

void AssemblerX86Shared::executableCopy(void* buffer) {
  masm.executableCopy(buffer);
}

void AssemblerX86Shared::processCodeLabels(uint8_t* rawCode) {
  for (const CodeLabel& label : codeLabels_) {
    Bind(rawCode, label);
  }
}

AssemblerX86Shared::Condition AssemblerX86Shared::InvertCondition(
    Condition cond) {
  switch (cond) {
    case Zero:
      return NonZero;
    case NonZero:
      return Zero;
    case LessThan:
      return GreaterThanOrEqual;
    case LessThanOrEqual:
      return GreaterThan;
    case GreaterThan:
      return LessThanOrEqual;
    case GreaterThanOrEqual:
      return LessThan;
    case Above:
      return BelowOrEqual;
    case AboveOrEqual:
      return Below;
    case Below:
      return AboveOrEqual;
    case BelowOrEqual:
      return Above;
    default:
      MOZ_CRASH("unexpected condition");
  }
}

AssemblerX86Shared::Condition AssemblerX86Shared::UnsignedCondition(
    Condition cond) {
  switch (cond) {
    case Zero:
    case NonZero:
      return cond;
    case LessThan:
    case Below:
      return Below;
    case LessThanOrEqual:
    case BelowOrEqual:
      return BelowOrEqual;
    case GreaterThan:
    case Above:
      return Above;
    case AboveOrEqual:
    case GreaterThanOrEqual:
      return AboveOrEqual;
    default:
      MOZ_CRASH("unexpected condition");
  }
}

AssemblerX86Shared::Condition AssemblerX86Shared::ConditionWithoutEqual(
    Condition cond) {
  switch (cond) {
    case LessThan:
    case LessThanOrEqual:
      return LessThan;
    case Below:
    case BelowOrEqual:
      return Below;
    case GreaterThan:
    case GreaterThanOrEqual:
      return GreaterThan;
    case Above:
    case AboveOrEqual:
      return Above;
    default:
      MOZ_CRASH("unexpected condition");
  }
}

AssemblerX86Shared::DoubleCondition AssemblerX86Shared::InvertCondition(
    DoubleCondition cond) {
  switch (cond) {
    case DoubleEqual:
      return DoubleNotEqualOrUnordered;
    case DoubleEqualOrUnordered:
      return DoubleNotEqual;
    case DoubleNotEqualOrUnordered:
      return DoubleEqual;
    case DoubleNotEqual:
      return DoubleEqualOrUnordered;
    case DoubleLessThan:
      return DoubleGreaterThanOrEqualOrUnordered;
    case DoubleLessThanOrUnordered:
      return DoubleGreaterThanOrEqual;
    case DoubleLessThanOrEqual:
      return DoubleGreaterThanOrUnordered;
    case DoubleLessThanOrEqualOrUnordered:
      return DoubleGreaterThan;
    case DoubleGreaterThan:
      return DoubleLessThanOrEqualOrUnordered;
    case DoubleGreaterThanOrUnordered:
      return DoubleLessThanOrEqual;
    case DoubleGreaterThanOrEqual:
      return DoubleLessThanOrUnordered;
    case DoubleGreaterThanOrEqualOrUnordered:
      return DoubleLessThan;
    default:
      MOZ_CRASH("unexpected condition");
  }
}

CPUInfo::SSEVersion CPUInfo::maxSSEVersion = UnknownSSE;
CPUInfo::SSEVersion CPUInfo::maxEnabledSSEVersion = UnknownSSE;
bool CPUInfo::avxPresent = false;
#ifdef ENABLE_WASM_AVX
bool CPUInfo::avxEnabled = true;
#else
bool CPUInfo::avxEnabled = false;
#endif
bool CPUInfo::popcntPresent = false;
bool CPUInfo::bmi1Present = false;
bool CPUInfo::bmi2Present = false;
bool CPUInfo::lzcntPresent = false;
bool CPUInfo::avx2Present = false;
bool CPUInfo::fmaPresent = false;

namespace js {
namespace jit {
bool CPUFlagsHaveBeenComputed() { return CPUInfo::FlagsHaveBeenComputed(); }
}  // namespace jit
}  // namespace js

static uintptr_t ReadXGETBV() {
  // We use a variety of low-level mechanisms to get at the xgetbv
  // instruction, including spelling out the xgetbv instruction as bytes,
  // because older compilers and assemblers may not recognize the instruction
  // by name.
  size_t xcr0EAX = 0;
#if defined(_XCR_XFEATURE_ENABLED_MASK)
  xcr0EAX = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
#elif defined(__GNUC__)
  // xgetbv returns its results in %eax and %edx, and for our purposes here,
  // we're only interested in the %eax value.
  asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0EAX) : "c"(0) : "%edx");
#elif defined(_MSC_VER) && defined(_M_IX86)
  __asm {
        xor ecx, ecx
        _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0
        mov xcr0EAX, eax
  }
#endif
  return xcr0EAX;
}

static void ReadCPUInfo(int* flagsEax, int* flagsEbx, int* flagsEcx,
                        int* flagsEdx) {
#ifdef _MSC_VER
  int cpuinfo[4];
  __cpuid(cpuinfo, *flagsEax);
  *flagsEax = cpuinfo[0];
  *flagsEbx = cpuinfo[1];
  *flagsEcx = cpuinfo[2];
  *flagsEdx = cpuinfo[3];
#elif defined(__GNUC__)
  // Some older 32-bits processors don't fill the ecx register with cpuid, so
  // clobber it before calling cpuid, so that there's no risk of picking
  // random bits indicating SSE3/SSE4 are present. Also make sure that it's
  // set to 0 as an input for BMI detection on all platforms.
  *flagsEcx = 0;
#  ifdef JS_CODEGEN_X64
  asm("cpuid;"
      : "+a"(*flagsEax), "=b"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
#  else
  // On x86, preserve ebx. The compiler needs it for PIC mode.
  asm("mov %%ebx, %%edi;"
      "cpuid;"
      "xchg %%edi, %%ebx;"
      : "+a"(*flagsEax), "=D"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
#  endif
#else
#  error "Unsupported compiler"
#endif
}

void CPUInfo::ComputeFlags() {
  MOZ_ASSERT(!FlagsHaveBeenComputed());

  int flagsEax = 1;
  int flagsEbx = 0;
  int flagsEcx = 0;
  int flagsEdx = 0;
  ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);

  static constexpr int SSEBit = 1 << 25;
  static constexpr int SSE2Bit = 1 << 26;
  static constexpr int SSE3Bit = 1 << 0;
  static constexpr int SSSE3Bit = 1 << 9;
  static constexpr int SSE41Bit = 1 << 19;
  static constexpr int SSE42Bit = 1 << 20;

  if (flagsEcx & SSE42Bit) {
    maxSSEVersion = SSE4_2;
  } else if (flagsEcx & SSE41Bit) {
    maxSSEVersion = SSE4_1;
  } else if (flagsEcx & SSSE3Bit) {
    maxSSEVersion = SSSE3;
  } else if (flagsEcx & SSE3Bit) {
    maxSSEVersion = SSE3;
  } else if (flagsEdx & SSE2Bit) {
    maxSSEVersion = SSE2;
  } else if (flagsEdx & SSEBit) {
    maxSSEVersion = SSE;
  } else {
    maxSSEVersion = NoSSE;
  }

  if (maxEnabledSSEVersion != UnknownSSE) {
    maxSSEVersion = std::min(maxSSEVersion, maxEnabledSSEVersion);
  }

  static constexpr int AVXBit = 1 << 28;
  static constexpr int XSAVEBit = 1 << 27;
  avxPresent = (flagsEcx & AVXBit) && (flagsEcx & XSAVEBit) && avxEnabled;

  // If the hardware supports AVX, check whether the OS supports it too.
  if (avxPresent) {
    size_t xcr0EAX = ReadXGETBV();
    static constexpr int xcr0SSEBit = 1 << 1;
    static constexpr int xcr0AVXBit = 1 << 2;
    avxPresent = (xcr0EAX & xcr0SSEBit) && (xcr0EAX & xcr0AVXBit);
  }

  // CMOV instruction are supposed to be supported by all CPU which have SSE2
  // enabled. While this might be true, this is not guaranteed by any
  // documentation, nor AMD, nor Intel.
  static constexpr int CMOVBit = 1 << 15;
  MOZ_RELEASE_ASSERT(flagsEdx & CMOVBit,
                     "CMOVcc instruction is not recognized by this CPU.");

  static constexpr int POPCNTBit = 1 << 23;
  popcntPresent = (flagsEcx & POPCNTBit);

  // Use the avxEnabled flag to enable/disable FMA.
  static constexpr int FMABit = 1 << 12;
  fmaPresent = (flagsEcx & FMABit) && avxEnabled;

  flagsEax = 0x80000001;
  ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);

  static constexpr int LZCNTBit = 1 << 5;
  lzcntPresent = (flagsEcx & LZCNTBit);

  flagsEax = 0x7;
  ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);

  static constexpr int BMI1Bit = 1 << 3;
  static constexpr int BMI2Bit = 1 << 8;
  static constexpr int AVX2Bit = 1 << 5;
  bmi1Present = (flagsEbx & BMI1Bit);
  bmi2Present = bmi1Present && (flagsEbx & BMI2Bit);
  avx2Present = avxPresent && (flagsEbx & AVX2Bit);

  MOZ_ASSERT(FlagsHaveBeenComputed());
}