path: root/js/src/jit/riscv64/Assembler-riscv64.cpp

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
 * vim: set ts=8 sts=2 et sw=2 tw=80:
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

// Copyright (c) 1994-2006 Sun Microsystems Inc.
// All Rights Reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// - Redistribution in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// - Neither the name of Sun Microsystems or the names of contributors may
// be used to endorse or promote products derived from this software without
// specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// The original source code covered by the above license above has been
// modified significantly by Google Inc.
// Copyright 2021 the V8 project authors. All rights reserved.
#include "jit/riscv64/Assembler-riscv64.h"

#include "mozilla/DebugOnly.h"
#include "mozilla/Maybe.h"

#include "gc/Marking.h"
#include "jit/AutoWritableJitCode.h"
#include "jit/ExecutableAllocator.h"
#include "jit/riscv64/disasm/Disasm-riscv64.h"
#include "vm/Realm.h"

using mozilla::DebugOnly;
namespace js {
namespace jit {

#define UNIMPLEMENTED_RISCV() MOZ_CRASH("RISC_V not implemented");

bool Assembler::FLAG_riscv_debug = false;

void Assembler::nop() { addi(ToRegister(0), ToRegister(0), 0); }

// Size of the instruction stream, in bytes.
size_t Assembler::size() const { return m_buffer.size(); }

bool Assembler::swapBuffer(wasm::Bytes& bytes) {
  // For now, specialize to the one use case. As long as wasm::Bytes is a
  // Vector, not a linked-list of chunks, there's not much we can do other
  // than copy.
  MOZ_ASSERT(bytes.empty());
  if (!bytes.resize(bytesNeeded())) {
    return false;
  }
  m_buffer.executableCopy(bytes.begin());
  return true;
}

// Size of the relocation table, in bytes.
size_t Assembler::jumpRelocationTableBytes() const {
  return jumpRelocations_.length();
}

size_t Assembler::dataRelocationTableBytes() const {
  return dataRelocations_.length();
}
// Size of the data table, in bytes.
size_t Assembler::bytesNeeded() const {
  return size() + jumpRelocationTableBytes() + dataRelocationTableBytes();
}

void Assembler::executableCopy(uint8_t* buffer) {
  MOZ_ASSERT(isFinished);
  m_buffer.executableCopy(buffer);
}

uint32_t Assembler::AsmPoolMaxOffset = 1024;

uint32_t Assembler::GetPoolMaxOffset() {
  static bool isSet = false;
  if (!isSet) {
    char* poolMaxOffsetStr = getenv("ASM_POOL_MAX_OFFSET");
    uint32_t poolMaxOffset;
    if (poolMaxOffsetStr &&
        sscanf(poolMaxOffsetStr, "%u", &poolMaxOffset) == 1) {
      AsmPoolMaxOffset = poolMaxOffset;
    }
    isSet = true;
  }
  return AsmPoolMaxOffset;
}

// Pool callbacks stuff:
void Assembler::InsertIndexIntoTag(uint8_t* load_, uint32_t index) {
  MOZ_CRASH("Unimplement");
}

void Assembler::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr) {
  MOZ_CRASH("Unimplement");
}

void Assembler::processCodeLabels(uint8_t* rawCode) {
  for (const CodeLabel& label : codeLabels_) {
    Bind(rawCode, label);
  }
}

void Assembler::WritePoolGuard(BufferOffset branch, Instruction* dest,
                               BufferOffset afterPool) {
  DEBUG_PRINTF("\tWritePoolGuard\n");
  int32_t off = afterPool.getOffset() - branch.getOffset();
  if (!is_int21(off) || !((off & 0x1) == 0)) {
    printf("%d\n", off);
    MOZ_CRASH("imm invalid");
  }
  // JAL encode is
  //   31    | 30    21  |  20     | 19     12  | 11 7 |  6   0 |
  // imm[20] | imm[10:1] | imm[11] | imm[19:12] |  rd  |  opcode|
  //   1           10         1           8         5       7
  //                   offset[20:1]               dest      JAL
  int32_t imm20 = (off & 0xff000) |          // bits 19-12
                  ((off & 0x800) << 9) |     // bit  11
                  ((off & 0x7fe) << 20) |    // bits 10-1
                  ((off & 0x100000) << 11);  // bit  20
  Instr instr = JAL | (imm20 & kImm20Mask);
  dest->SetInstructionBits(instr);
  DEBUG_PRINTF("%p(%x): ", dest, branch.getOffset());
  disassembleInstr(dest->InstructionBits(), JitSpew_Codegen);
}

void Assembler::WritePoolHeader(uint8_t* start, Pool* p, bool isNatural) {
  static_assert(sizeof(PoolHeader) == 4);

  // Get the total size of the pool.
  const uintptr_t totalPoolSize = sizeof(PoolHeader) + p->getPoolSize();
  const uintptr_t totalPoolInstructions = totalPoolSize / kInstrSize;

  MOZ_ASSERT((totalPoolSize & 0x3) == 0);
  MOZ_ASSERT(totalPoolInstructions < (1 << 15));

  PoolHeader header(totalPoolInstructions, isNatural);
  *(PoolHeader*)start = header;
}

void Assembler::copyJumpRelocationTable(uint8_t* dest) {
  if (jumpRelocations_.length()) {
    memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
  }
}

void Assembler::copyDataRelocationTable(uint8_t* dest) {
  if (dataRelocations_.length()) {
    memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
  }
}

void Assembler::RV_li(Register rd, int64_t imm) {
  UseScratchRegisterScope temps(this);
  if (RecursiveLiCount(imm) > GeneralLiCount(imm, temps.hasAvailable())) {
    GeneralLi(rd, imm);
  } else {
    RecursiveLi(rd, imm);
  }
}

int Assembler::RV_li_count(int64_t imm, bool is_get_temp_reg) {
  if (RecursiveLiCount(imm) > GeneralLiCount(imm, is_get_temp_reg)) {
    return GeneralLiCount(imm, is_get_temp_reg);
  } else {
    return RecursiveLiCount(imm);
  }
}

void Assembler::GeneralLi(Register rd, int64_t imm) {
  // 64-bit imm is put in the register rd.
  // In most cases the imm is 32 bit and 2 instructions are generated. If a
  // temporary register is available, in the worst case, 6 instructions are
  // generated for a full 64-bit immediate. If temporay register is not
  // available the maximum will be 8 instructions. If imm is more than 32 bits
  // and a temp register is available, imm is divided into two 32-bit parts,
  // low_32 and up_32. Each part is built in a separate register. low_32 is
  // built before up_32. If low_32 is negative (upper 32 bits are 1), 0xffffffff
  // is subtracted from up_32 before up_32 is built. This compensates for 32
  // bits of 1's in the lower when the two registers are added. If no temp is
  // available, the upper 32 bit is built in rd, and the lower 32 bits are
  // devided to 3 parts (11, 11, and 10 bits). The parts are shifted and added
  // to the upper part built in rd.
  if (is_int32(imm + 0x800)) {
    // 32-bit case. Maximum of 2 instructions generated
    int64_t high_20 = ((imm + 0x800) >> 12);
    int64_t low_12 = imm << 52 >> 52;
    if (high_20) {
      lui(rd, (int32_t)high_20);
      if (low_12) {
        addi(rd, rd, low_12);
      }
    } else {
      addi(rd, zero_reg, low_12);
    }
    return;
  } else {
    UseScratchRegisterScope temps(this);
    BlockTrampolinePoolScope block_trampoline_pool(this, 8);
    // 64-bit case: divide imm into two 32-bit parts, upper and lower
    int64_t up_32 = imm >> 32;
    int64_t low_32 = imm & 0xffffffffull;
    Register temp_reg = rd;
    // Check if a temporary register is available
    if (up_32 == 0 || low_32 == 0) {
      // No temp register is needed
    } else {
      temp_reg = temps.hasAvailable() ? temps.Acquire() : InvalidReg;
    }
    if (temp_reg != InvalidReg) {
      // keep track of hardware behavior for lower part in sim_low
      int64_t sim_low = 0;
      // Build lower part
      if (low_32 != 0) {
        int64_t high_20 = ((low_32 + 0x800) >> 12);
        int64_t low_12 = low_32 & 0xfff;
        if (high_20) {
          // Adjust to 20 bits for the case of overflow
          high_20 &= 0xfffff;
          sim_low = ((high_20 << 12) << 32) >> 32;
          lui(rd, (int32_t)high_20);
          if (low_12) {
            sim_low += (low_12 << 52 >> 52) | low_12;
            addi(rd, rd, low_12);
          }
        } else {
          sim_low = low_12;
          ori(rd, zero_reg, low_12);
        }
      }
      if (sim_low & 0x100000000) {
        // Bit 31 is 1. Either an overflow or a negative 64 bit
        if (up_32 == 0) {
          // Positive number, but overflow because of the add 0x800
          slli(rd, rd, 32);
          srli(rd, rd, 32);
          return;
        }
        // low_32 is a negative 64 bit after the build
        up_32 = (up_32 - 0xffffffff) & 0xffffffff;
      }
      if (up_32 == 0) {
        return;
      }
      // Build upper part in a temporary register
      if (low_32 == 0) {
        // Build upper part in rd
        temp_reg = rd;
      }
      int64_t high_20 = (up_32 + 0x800) >> 12;
      int64_t low_12 = up_32 & 0xfff;
      if (high_20) {
        // Adjust to 20 bits for the case of overflow
        high_20 &= 0xfffff;
        lui(temp_reg, (int32_t)high_20);
        if (low_12) {
          addi(temp_reg, temp_reg, low_12);
        }
      } else {
        ori(temp_reg, zero_reg, low_12);
      }
      // Put it at the bgining of register
      slli(temp_reg, temp_reg, 32);
      if (low_32 != 0) {
        add(rd, rd, temp_reg);
      }
      return;
    }
    // No temp register. Build imm in rd.
    // Build upper 32 bits first in rd. Divide lower 32 bits parts and add
    // parts to the upper part by doing shift and add.
    // First build upper part in rd.
    int64_t high_20 = (up_32 + 0x800) >> 12;
    int64_t low_12 = up_32 & 0xfff;
    if (high_20) {
      // Adjust to 20 bits for the case of overflow
      high_20 &= 0xfffff;
      lui(rd, (int32_t)high_20);
      if (low_12) {
        addi(rd, rd, low_12);
      }
    } else {
      ori(rd, zero_reg, low_12);
    }
    // upper part already in rd. Each part to be added to rd, has maximum of 11
    // bits, and always starts with a 1. rd is shifted by the size of the part
    // plus the number of zeros between the parts. Each part is added after the
    // left shift.
    uint32_t mask = 0x80000000;
    int32_t shift_val = 0;
    int32_t i;
    for (i = 0; i < 32; i++) {
      if ((low_32 & mask) == 0) {
        mask >>= 1;
        shift_val++;
        if (i == 31) {
          // rest is zero
          slli(rd, rd, shift_val);
        }
        continue;
      }
      // The first 1 seen
      int32_t part;
      if ((i + 11) < 32) {
        // Pick 11 bits
        part = ((uint32_t)(low_32 << i) >> i) >> (32 - (i + 11));
        slli(rd, rd, shift_val + 11);
        ori(rd, rd, part);
        i += 10;
        mask >>= 11;
      } else {
        part = (uint32_t)(low_32 << i) >> i;
        slli(rd, rd, shift_val + (32 - i));
        ori(rd, rd, part);
        break;
      }
      shift_val = 0;
    }
  }
}

int Assembler::GeneralLiCount(int64_t imm, bool is_get_temp_reg) {
  int count = 0;
  // imitate Assembler::RV_li
  if (is_int32(imm + 0x800)) {
    // 32-bit case. Maximum of 2 instructions generated
    int64_t high_20 = ((imm + 0x800) >> 12);
    int64_t low_12 = imm << 52 >> 52;
    if (high_20) {
      count++;
      if (low_12) {
        count++;
      }
    } else {
      count++;
    }
    return count;
  } else {
    // 64-bit case: divide imm into two 32-bit parts, upper and lower
    int64_t up_32 = imm >> 32;
    int64_t low_32 = imm & 0xffffffffull;
    // Check if a temporary register is available
    if (is_get_temp_reg) {
      // keep track of hardware behavior for lower part in sim_low
      int64_t sim_low = 0;
      // Build lower part
      if (low_32 != 0) {
        int64_t high_20 = ((low_32 + 0x800) >> 12);
        int64_t low_12 = low_32 & 0xfff;
        if (high_20) {
          // Adjust to 20 bits for the case of overflow
          high_20 &= 0xfffff;
          sim_low = ((high_20 << 12) << 32) >> 32;
          count++;
          if (low_12) {
            sim_low += (low_12 << 52 >> 52) | low_12;
            count++;
          }
        } else {
          sim_low = low_12;
          count++;
        }
      }
      if (sim_low & 0x100000000) {
        // Bit 31 is 1. Either an overflow or a negative 64 bit
        if (up_32 == 0) {
          // Positive number, but overflow because of the add 0x800
          count++;
          count++;
          return count;
        }
        // low_32 is a negative 64 bit after the build
        up_32 = (up_32 - 0xffffffff) & 0xffffffff;
      }
      if (up_32 == 0) {
        return count;
      }
      int64_t high_20 = (up_32 + 0x800) >> 12;
      int64_t low_12 = up_32 & 0xfff;
      if (high_20) {
        // Adjust to 20 bits for the case of overflow
        high_20 &= 0xfffff;
        count++;
        if (low_12) {
          count++;
        }
      } else {
        count++;
      }
      // Put it at the bgining of register
      count++;
      if (low_32 != 0) {
        count++;
      }
      return count;
    }
    // No temp register. Build imm in rd.
    // Build upper 32 bits first in rd. Divide lower 32 bits parts and add
    // parts to the upper part by doing shift and add.
    // First build upper part in rd.
    int64_t high_20 = (up_32 + 0x800) >> 12;
    int64_t low_12 = up_32 & 0xfff;
    if (high_20) {
      // Adjust to 20 bits for the case of overflow
      high_20 &= 0xfffff;
      count++;
      if (low_12) {
        count++;
      }
    } else {
      count++;
    }
    // upper part already in rd. Each part to be added to rd, has maximum of 11
    // bits, and always starts with a 1. rd is shifted by the size of the part
    // plus the number of zeros between the parts. Each part is added after the
    // left shift.
    uint32_t mask = 0x80000000;
    int32_t i;
    for (i = 0; i < 32; i++) {
      if ((low_32 & mask) == 0) {
        mask >>= 1;
        if (i == 31) {
          // rest is zero
          count++;
        }
        continue;
      }
      // The first 1 seen
      if ((i + 11) < 32) {
        // Pick 11 bits
        count++;
        count++;
        i += 10;
        mask >>= 11;
      } else {
        count++;
        count++;
        break;
      }
    }
  }
  return count;
}

void Assembler::li_ptr(Register rd, int64_t imm) {
  m_buffer.enterNoNops();
  m_buffer.assertNoPoolAndNoNops();
  // Initialize rd with an address
  // Pointers are 48 bits
  // 6 fixed instructions are generated
  DEBUG_PRINTF("li_ptr(%d, %lx <%ld>)\n", ToNumber(rd), imm, imm);
  MOZ_ASSERT((imm & 0xfff0000000000000ll) == 0);
  int64_t a6 = imm & 0x3f;                      // bits 0:5. 6 bits
  int64_t b11 = (imm >> 6) & 0x7ff;             // bits 6:11. 11 bits
  int64_t high_31 = (imm >> 17) & 0x7fffffff;   // 31 bits
  int64_t high_20 = ((high_31 + 0x800) >> 12);  // 19 bits
  int64_t low_12 = high_31 & 0xfff;             // 12 bits
  lui(rd, (int32_t)high_20);
  addi(rd, rd, low_12);  // 31 bits in rd.
  slli(rd, rd, 11);      // Space for next 11 bis
  ori(rd, rd, b11);      // 11 bits are put in. 42 bit in rd
  slli(rd, rd, 6);       // Space for next 6 bits
  ori(rd, rd, a6);       // 6 bits are put in. 48 bis in rd
  m_buffer.leaveNoNops();
}

void Assembler::li_constant(Register rd, int64_t imm) {
  m_buffer.enterNoNops();
  m_buffer.assertNoPoolAndNoNops();
  DEBUG_PRINTF("li_constant(%d, %lx <%ld>)\n", ToNumber(rd), imm, imm);
  lui(rd, (imm + (1LL << 47) + (1LL << 35) + (1LL << 23) + (1LL << 11)) >>
              48);  // Bits 63:48
  addiw(rd, rd,
        (imm + (1LL << 35) + (1LL << 23) + (1LL << 11)) << 16 >>
            52);  // Bits 47:36
  slli(rd, rd, 12);
  addi(rd, rd, (imm + (1LL << 23) + (1LL << 11)) << 28 >> 52);  // Bits 35:24
  slli(rd, rd, 12);
  addi(rd, rd, (imm + (1LL << 11)) << 40 >> 52);  // Bits 23:12
  slli(rd, rd, 12);
  addi(rd, rd, imm << 52 >> 52);  // Bits 11:0
  m_buffer.leaveNoNops();
}

ABIArg ABIArgGenerator::next(MIRType type) {
  switch (type) {
    case MIRType::Int32:
    case MIRType::Int64:
    case MIRType::Pointer:
    case MIRType::WasmAnyRef:
    case MIRType::StackResults: {
      if (intRegIndex_ == NumIntArgRegs) {
        current_ = ABIArg(stackOffset_);
        stackOffset_ += sizeof(uintptr_t);
        break;
      }
      current_ = ABIArg(Register::FromCode(intRegIndex_ + a0.encoding()));
      intRegIndex_++;
      break;
    }
    case MIRType::Float32:
    case MIRType::Double: {
      if (floatRegIndex_ == NumFloatArgRegs) {
        current_ = ABIArg(stackOffset_);
        stackOffset_ += sizeof(double);
        break;
      }
      current_ = ABIArg(FloatRegister(
          FloatRegisters::Encoding(floatRegIndex_ + fa0.encoding()),
          type == MIRType::Double ? FloatRegisters::Double
                                  : FloatRegisters::Single));
      floatRegIndex_++;
      break;
    }
    case MIRType::Simd128: {
      MOZ_CRASH("RISCV64 does not support simd yet.");
      break;
    }
    default:
      MOZ_CRASH("Unexpected argument type");
  }
  return current_;
}

bool Assembler::oom() const {
  return AssemblerShared::oom() || m_buffer.oom() || jumpRelocations_.oom() ||
         dataRelocations_.oom() || !enoughLabelCache_;
}

int Assembler::disassembleInstr(Instr instr, bool enable_spew) {
  if (!FLAG_riscv_debug && !enable_spew) return -1;
  disasm::NameConverter converter;
  disasm::Disassembler disasm(converter);
  EmbeddedVector<char, 128> disasm_buffer;

  int size =
      disasm.InstructionDecode(disasm_buffer, reinterpret_cast<byte*>(&instr));
  DEBUG_PRINTF("%s\n", disasm_buffer.start());
  if (enable_spew) {
    JitSpew(JitSpew_Codegen, "%s", disasm_buffer.start());
  }
  return size;
}

uintptr_t Assembler::target_address_at(Instruction* pc) {
  Instruction* instr0 = pc;
  DEBUG_PRINTF("target_address_at: pc: 0x%p\t", instr0);
  Instruction* instr1 = pc + 1 * kInstrSize;
  Instruction* instr2 = pc + 2 * kInstrSize;
  Instruction* instr3 = pc + 3 * kInstrSize;
  Instruction* instr4 = pc + 4 * kInstrSize;
  Instruction* instr5 = pc + 5 * kInstrSize;

  // Interpret instructions for address generated by li: See listing in
  // Assembler::set_target_address_at() just below.
  if (IsLui(*reinterpret_cast<Instr*>(instr0)) &&
      IsAddi(*reinterpret_cast<Instr*>(instr1)) &&
      IsSlli(*reinterpret_cast<Instr*>(instr2)) &&
      IsOri(*reinterpret_cast<Instr*>(instr3)) &&
      IsSlli(*reinterpret_cast<Instr*>(instr4)) &&
      IsOri(*reinterpret_cast<Instr*>(instr5))) {
    // Assemble the 64 bit value.
    int64_t addr = (int64_t)(instr0->Imm20UValue() << kImm20Shift) +
                   (int64_t)instr1->Imm12Value();
    MOZ_ASSERT(instr2->Imm12Value() == 11);
    addr <<= 11;
    addr |= (int64_t)instr3->Imm12Value();
    MOZ_ASSERT(instr4->Imm12Value() == 6);
    addr <<= 6;
    addr |= (int64_t)instr5->Imm12Value();

    DEBUG_PRINTF("addr: %lx\n", addr);
    return static_cast<uintptr_t>(addr);
  }
  // We should never get here, force a bad address if we do.
  MOZ_CRASH("RISC-V  UNREACHABLE");
}

void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
                                        ImmPtr newValue, ImmPtr expectedValue) {
  PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value),
                          PatchedImmPtr(expectedValue.value));
}

void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
                                        PatchedImmPtr newValue,
                                        PatchedImmPtr expectedValue) {
  Instruction* inst = (Instruction*)label.raw();

  // Extract old Value
  DebugOnly<uint64_t> value = Assembler::ExtractLoad64Value(inst);
  MOZ_ASSERT(value == uint64_t(expectedValue.value));

  // Replace with new value
  Assembler::UpdateLoad64Value(inst, uint64_t(newValue.value));
}

uint64_t Assembler::ExtractLoad64Value(Instruction* inst0) {
  DEBUG_PRINTF("\tExtractLoad64Value: \tpc:%p ", inst0);
  if (IsJal(*reinterpret_cast<Instr*>(inst0))) {
    int offset = inst0->Imm20JValue();
    inst0 = inst0 + offset;
  }
  Instruction* instr1 = inst0 + 1 * kInstrSize;
  if (IsAddiw(*reinterpret_cast<Instr*>(instr1))) {
    // Li64
    Instruction* instr2 = inst0 + 2 * kInstrSize;
    Instruction* instr3 = inst0 + 3 * kInstrSize;
    Instruction* instr4 = inst0 + 4 * kInstrSize;
    Instruction* instr5 = inst0 + 5 * kInstrSize;
    Instruction* instr6 = inst0 + 6 * kInstrSize;
    Instruction* instr7 = inst0 + 7 * kInstrSize;
    if (IsLui(*reinterpret_cast<Instr*>(inst0)) &&
        IsAddiw(*reinterpret_cast<Instr*>(instr1)) &&
        IsSlli(*reinterpret_cast<Instr*>(instr2)) &&
        IsAddi(*reinterpret_cast<Instr*>(instr3)) &&
        IsSlli(*reinterpret_cast<Instr*>(instr4)) &&
        IsAddi(*reinterpret_cast<Instr*>(instr5)) &&
        IsSlli(*reinterpret_cast<Instr*>(instr6)) &&
        IsAddi(*reinterpret_cast<Instr*>(instr7))) {
      int64_t imm = (int64_t)(inst0->Imm20UValue() << kImm20Shift) +
                    (int64_t)instr1->Imm12Value();
      MOZ_ASSERT(instr2->Imm12Value() == 12);
      imm <<= 12;
      imm += (int64_t)instr3->Imm12Value();
      MOZ_ASSERT(instr4->Imm12Value() == 12);
      imm <<= 12;
      imm += (int64_t)instr5->Imm12Value();
      MOZ_ASSERT(instr6->Imm12Value() == 12);
      imm <<= 12;
      imm += (int64_t)instr7->Imm12Value();
      DEBUG_PRINTF("imm:%lx\n", imm);
      return imm;
    } else {
      FLAG_riscv_debug = true;
      disassembleInstr(inst0->InstructionBits());
      disassembleInstr(instr1->InstructionBits());
      disassembleInstr(instr2->InstructionBits());
      disassembleInstr(instr3->InstructionBits());
      disassembleInstr(instr4->InstructionBits());
      disassembleInstr(instr5->InstructionBits());
      disassembleInstr(instr6->InstructionBits());
      disassembleInstr(instr7->InstructionBits());
      MOZ_CRASH();
    }
  } else {
    DEBUG_PRINTF("\n");
    Instruction* instrf1 = (inst0 - 1 * kInstrSize);
    Instruction* instr2 = inst0 + 2 * kInstrSize;
    Instruction* instr3 = inst0 + 3 * kInstrSize;
    Instruction* instr4 = inst0 + 4 * kInstrSize;
    Instruction* instr5 = inst0 + 5 * kInstrSize;
    Instruction* instr6 = inst0 + 6 * kInstrSize;
    Instruction* instr7 = inst0 + 7 * kInstrSize;
    disassembleInstr(instrf1->InstructionBits());
    disassembleInstr(inst0->InstructionBits());
    disassembleInstr(instr1->InstructionBits());
    disassembleInstr(instr2->InstructionBits());
    disassembleInstr(instr3->InstructionBits());
    disassembleInstr(instr4->InstructionBits());
    disassembleInstr(instr5->InstructionBits());
    disassembleInstr(instr6->InstructionBits());
    disassembleInstr(instr7->InstructionBits());
    MOZ_ASSERT(IsAddi(*reinterpret_cast<Instr*>(instr1)));
    // Li48
    return target_address_at(inst0);
  }
}

void Assembler::UpdateLoad64Value(Instruction* pc, uint64_t value) {
  DEBUG_PRINTF("\tUpdateLoad64Value: pc: %p\tvalue: %lx\n", pc, value);
  Instruction* instr1 = pc + 1 * kInstrSize;
  if (IsJal(*reinterpret_cast<Instr*>(pc))) {
    pc = pc + pc->Imm20JValue();
    instr1 = pc + 1 * kInstrSize;
  }
  if (IsAddiw(*reinterpret_cast<Instr*>(instr1))) {
    Instruction* instr0 = pc;
    Instruction* instr2 = pc + 2 * kInstrSize;
    Instruction* instr3 = pc + 3 * kInstrSize;
    Instruction* instr4 = pc + 4 * kInstrSize;
    Instruction* instr5 = pc + 5 * kInstrSize;
    Instruction* instr6 = pc + 6 * kInstrSize;
    Instruction* instr7 = pc + 7 * kInstrSize;
    MOZ_ASSERT(IsLui(*reinterpret_cast<Instr*>(pc)) &&
               IsAddiw(*reinterpret_cast<Instr*>(instr1)) &&
               IsSlli(*reinterpret_cast<Instr*>(instr2)) &&
               IsAddi(*reinterpret_cast<Instr*>(instr3)) &&
               IsSlli(*reinterpret_cast<Instr*>(instr4)) &&
               IsAddi(*reinterpret_cast<Instr*>(instr5)) &&
               IsSlli(*reinterpret_cast<Instr*>(instr6)) &&
               IsAddi(*reinterpret_cast<Instr*>(instr7)));
    // lui(rd, (imm + (1LL << 47) + (1LL << 35) + (1LL << 23) + (1LL << 11)) >>
    //             48);  // Bits 63:48
    // addiw(rd, rd,
    //       (imm + (1LL << 35) + (1LL << 23) + (1LL << 11)) << 16 >>
    //           52);  // Bits 47:36
    // slli(rd, rd, 12);
    // addi(rd, rd, (imm + (1LL << 23) + (1LL << 11)) << 28 >> 52);  // Bits
    // 35:24 slli(rd, rd, 12); addi(rd, rd, (imm + (1LL << 11)) << 40 >> 52); //
    // Bits 23:12 slli(rd, rd, 12); addi(rd, rd, imm << 52 >> 52);  // Bits 11:0
    *reinterpret_cast<Instr*>(instr0) &= 0xfff;
    *reinterpret_cast<Instr*>(instr0) |=
        (((value + (1LL << 47) + (1LL << 35) + (1LL << 23) + (1LL << 11)) >> 48)
         << 12);
    *reinterpret_cast<Instr*>(instr1) &= 0xfffff;
    *reinterpret_cast<Instr*>(instr1) |=
        (((value + (1LL << 35) + (1LL << 23) + (1LL << 11)) << 16 >> 52) << 20);
    *reinterpret_cast<Instr*>(instr3) &= 0xfffff;
    *reinterpret_cast<Instr*>(instr3) |=
        (((value + (1LL << 23) + (1LL << 11)) << 28 >> 52) << 20);
    *reinterpret_cast<Instr*>(instr5) &= 0xfffff;
    *reinterpret_cast<Instr*>(instr5) |=
        (((value + (1LL << 11)) << 40 >> 52) << 20);
    *reinterpret_cast<Instr*>(instr7) &= 0xfffff;
    *reinterpret_cast<Instr*>(instr7) |= ((value << 52 >> 52) << 20);
    disassembleInstr(instr0->InstructionBits());
    disassembleInstr(instr1->InstructionBits());
    disassembleInstr(instr2->InstructionBits());
    disassembleInstr(instr3->InstructionBits());
    disassembleInstr(instr4->InstructionBits());
    disassembleInstr(instr5->InstructionBits());
    disassembleInstr(instr6->InstructionBits());
    disassembleInstr(instr7->InstructionBits());
    MOZ_ASSERT(ExtractLoad64Value(pc) == value);
  } else {
    Instruction* instr0 = pc;
    Instruction* instr2 = pc + 2 * kInstrSize;
    Instruction* instr3 = pc + 3 * kInstrSize;
    Instruction* instr4 = pc + 4 * kInstrSize;
    Instruction* instr5 = pc + 5 * kInstrSize;
    Instruction* instr6 = pc + 6 * kInstrSize;
    Instruction* instr7 = pc + 7 * kInstrSize;
    disassembleInstr(instr0->InstructionBits());
    disassembleInstr(instr1->InstructionBits());
    disassembleInstr(instr2->InstructionBits());
    disassembleInstr(instr3->InstructionBits());
    disassembleInstr(instr4->InstructionBits());
    disassembleInstr(instr5->InstructionBits());
    disassembleInstr(instr6->InstructionBits());
    disassembleInstr(instr7->InstructionBits());
    MOZ_ASSERT(IsAddi(*reinterpret_cast<Instr*>(instr1)));
    set_target_value_at(pc, value);
  }
}

void Assembler::set_target_value_at(Instruction* pc, uint64_t target) {
  DEBUG_PRINTF("\tset_target_value_at: pc: %p\ttarget: %lx\n", pc, target);
  uint32_t* p = reinterpret_cast<uint32_t*>(pc);
  MOZ_ASSERT((target & 0xffff000000000000ll) == 0);
#ifdef DEBUG
  // Check we have the result from a li macro-instruction.
  Instruction* instr0 = pc;
  Instruction* instr1 = pc + 1 * kInstrSize;
  Instruction* instr3 = pc + 3 * kInstrSize;
  Instruction* instr5 = pc + 5 * kInstrSize;
  MOZ_ASSERT(IsLui(*reinterpret_cast<Instr*>(instr0)) &&
             IsAddi(*reinterpret_cast<Instr*>(instr1)) &&
             IsOri(*reinterpret_cast<Instr*>(instr3)) &&
             IsOri(*reinterpret_cast<Instr*>(instr5)));
#endif
  int64_t a6 = target & 0x3f;                     // bits 0:6. 6 bits
  int64_t b11 = (target >> 6) & 0x7ff;            // bits 6:11. 11 bits
  int64_t high_31 = (target >> 17) & 0x7fffffff;  // 31 bits
  int64_t high_20 = ((high_31 + 0x800) >> 12);    // 19 bits
  int64_t low_12 = high_31 & 0xfff;               // 12 bits
  *p = *p & 0xfff;
  *p = *p | ((int32_t)high_20 << 12);
  *(p + 1) = *(p + 1) & 0xfffff;
  *(p + 1) = *(p + 1) | ((int32_t)low_12 << 20);
  *(p + 2) = *(p + 2) & 0xfffff;
  *(p + 2) = *(p + 2) | (11 << 20);
  *(p + 3) = *(p + 3) & 0xfffff;
  *(p + 3) = *(p + 3) | ((int32_t)b11 << 20);
  *(p + 4) = *(p + 4) & 0xfffff;
  *(p + 4) = *(p + 4) | (6 << 20);
  *(p + 5) = *(p + 5) & 0xfffff;
  *(p + 5) = *(p + 5) | ((int32_t)a6 << 20);
  MOZ_ASSERT(target_address_at(pc) == target);
}

void Assembler::WriteLoad64Instructions(Instruction* inst0, Register reg,
                                        uint64_t value) {
  DEBUG_PRINTF("\tWriteLoad64Instructions\n");
  // Initialize rd with an address
  // Pointers are 48 bits
  // 6 fixed instructions are generated
  MOZ_ASSERT((value & 0xfff0000000000000ll) == 0);
  int64_t a6 = value & 0x3f;                     // bits 0:5. 6 bits
  int64_t b11 = (value >> 6) & 0x7ff;            // bits 6:11. 11 bits
  int64_t high_31 = (value >> 17) & 0x7fffffff;  // 31 bits
  int64_t high_20 = ((high_31 + 0x800) >> 12);   // 19 bits
  int64_t low_12 = high_31 & 0xfff;              // 12 bits
  Instr lui_ = LUI | (reg.code() << kRdShift) |
               ((int32_t)high_20 << kImm20Shift);  // lui(rd, (int32_t)high_20);
  *reinterpret_cast<Instr*>(inst0) = lui_;

  Instr addi_ =
      OP_IMM | (reg.code() << kRdShift) | (0b000 << kFunct3Shift) |
      (reg.code() << kRs1Shift) |
      (low_12 << kImm12Shift);  // addi(rd, rd, low_12);  // 31 bits in rd.
  *reinterpret_cast<Instr*>(inst0 + 1 * kInstrSize) = addi_;

  Instr slli_ =
      OP_IMM | (reg.code() << kRdShift) | (0b001 << kFunct3Shift) |
      (reg.code() << kRs1Shift) |
      (11 << kImm12Shift);  // slli(rd, rd, 11);      // Space for next 11 bis
  *reinterpret_cast<Instr*>(inst0 + 2 * kInstrSize) = slli_;

  Instr ori_b11 = OP_IMM | (reg.code() << kRdShift) | (0b110 << kFunct3Shift) |
                  (reg.code() << kRs1Shift) |
                  (b11 << kImm12Shift);  // ori(rd, rd, b11);      // 11 bits
                                         // are put in. 42 bit in rd
  *reinterpret_cast<Instr*>(inst0 + 3 * kInstrSize) = ori_b11;

  slli_ = OP_IMM | (reg.code() << kRdShift) | (0b001 << kFunct3Shift) |
          (reg.code() << kRs1Shift) |
          (6 << kImm12Shift);  // slli(rd, rd, 6);      // Space for next 11 bis
  *reinterpret_cast<Instr*>(inst0 + 4 * kInstrSize) =
      slli_;  // slli(rd, rd, 6);       // Space for next 6 bits

  Instr ori_a6 = OP_IMM | (reg.code() << kRdShift) | (0b110 << kFunct3Shift) |
                 (reg.code() << kRs1Shift) |
                 (a6 << kImm12Shift);  // ori(rd, rd, a6);       // 6 bits are
                                       // put in. 48 bis in rd
  *reinterpret_cast<Instr*>(inst0 + 5 * kInstrSize) = ori_a6;
  disassembleInstr((inst0 + 0 * kInstrSize)->InstructionBits());
  disassembleInstr((inst0 + 1 * kInstrSize)->InstructionBits());
  disassembleInstr((inst0 + 2 * kInstrSize)->InstructionBits());
  disassembleInstr((inst0 + 3 * kInstrSize)->InstructionBits());
  disassembleInstr((inst0 + 4 * kInstrSize)->InstructionBits());
  disassembleInstr((inst0 + 5 * kInstrSize)->InstructionBits());
  disassembleInstr((inst0 + 6 * kInstrSize)->InstructionBits());
  MOZ_ASSERT(ExtractLoad64Value(inst0) == value);
}

// This just stomps over memory with 32 bits of raw data. Its purpose is to
// overwrite the call of JITed code with 32 bits worth of an offset. This will
// is only meant to function on code that has been invalidated, so it should
// be totally safe. Since that instruction will never be executed again, a
// ICache flush should not be necessary
void Assembler::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) {
  // Raw is going to be the return address.
  uint32_t* raw = (uint32_t*)label.raw();
  // Overwrite the 4 bytes before the return address, which will
  // end up being the call instruction.
  *(raw - 1) = imm.value;
}

void Assembler::target_at_put(BufferOffset pos, BufferOffset target_pos,
                              bool trampoline) {
  if (m_buffer.oom()) {
    return;
  }
  DEBUG_PRINTF("\ttarget_at_put: %p (%d) to %p (%d)\n",
               reinterpret_cast<Instr*>(editSrc(pos)), pos.getOffset(),
               reinterpret_cast<Instr*>(editSrc(pos)) + target_pos.getOffset() -
                   pos.getOffset(),
               target_pos.getOffset());
  Instruction* instruction = editSrc(pos);
  Instr instr = instruction->InstructionBits();
  switch (instruction->InstructionOpcodeType()) {
    case BRANCH: {
      instr = SetBranchOffset(pos.getOffset(), target_pos.getOffset(), instr);
      instr_at_put(pos, instr);
    } break;
    case JAL: {
      MOZ_ASSERT(IsJal(instr));
      instr = SetJalOffset(pos.getOffset(), target_pos.getOffset(), instr);
      instr_at_put(pos, instr);
    } break;
    case LUI: {
      set_target_value_at(instruction,
                          reinterpret_cast<uintptr_t>(editSrc(target_pos)));
    } break;
    case AUIPC: {
      Instr instr_auipc = instr;
      Instr instr_I =
          editSrc(BufferOffset(pos.getOffset() + 4))->InstructionBits();
      MOZ_ASSERT(IsJalr(instr_I) || IsAddi(instr_I));

      intptr_t offset = target_pos.getOffset() - pos.getOffset();
      if (is_int21(offset) && IsJalr(instr_I) && trampoline) {
        MOZ_ASSERT(is_int21(offset) && ((offset & 1) == 0));
        Instr instr = JAL;
        instr = SetJalOffset(pos.getOffset(), target_pos.getOffset(), instr);
        MOZ_ASSERT(IsJal(instr));
        MOZ_ASSERT(JumpOffset(instr) == offset);
        instr_at_put(pos, instr);
        instr_at_put(BufferOffset(pos.getOffset() + 4), kNopByte);
      } else {
        MOZ_RELEASE_ASSERT(is_int32(offset + 0x800));
        MOZ_ASSERT(instruction->RdValue() ==
                   editSrc(BufferOffset(pos.getOffset() + 4))->Rs1Value());
        int32_t Hi20 = (((int32_t)offset + 0x800) >> 12);
        int32_t Lo12 = (int32_t)offset << 20 >> 20;

        instr_auipc =
            (instr_auipc & ~kImm31_12Mask) | ((Hi20 & kImm19_0Mask) << 12);
        instr_at_put(pos, instr_auipc);

        const int kImm31_20Mask = ((1 << 12) - 1) << 20;
        const int kImm11_0Mask = ((1 << 12) - 1);
        instr_I = (instr_I & ~kImm31_20Mask) | ((Lo12 & kImm11_0Mask) << 20);
        instr_at_put(BufferOffset(pos.getOffset() + 4), instr_I);
      }
    } break;
    default:
      UNIMPLEMENTED_RISCV();
      break;
  }
}

const int kEndOfChain = -1;
const int32_t kEndOfJumpChain = 0;

int Assembler::target_at(BufferOffset pos, bool is_internal) {
  if (oom()) {
    return kEndOfChain;
  }
  Instruction* instruction = editSrc(pos);
  Instruction* instruction2 = nullptr;
  if (IsAuipc(instruction->InstructionBits())) {
    instruction2 = editSrc(BufferOffset(pos.getOffset() + kInstrSize));
  }
  return target_at(instruction, pos, is_internal, instruction2);
}

int Assembler::target_at(Instruction* instruction, BufferOffset pos,
                         bool is_internal, Instruction* instruction2) {
  DEBUG_PRINTF("\t target_at: %p(%x)\n\t",
               reinterpret_cast<Instr*>(instruction), pos.getOffset());
  disassembleInstr(instruction->InstructionBits());
  Instr instr = instruction->InstructionBits();
  switch (instruction->InstructionOpcodeType()) {
    case BRANCH: {
      int32_t imm13 = BranchOffset(instr);
      if (imm13 == kEndOfJumpChain) {
        // EndOfChain sentinel is returned directly, not relative to pc or pos.
        return kEndOfChain;
      } else {
        DEBUG_PRINTF("\t target_at: %d %d\n", imm13, pos.getOffset() + imm13);
        return pos.getOffset() + imm13;
      }
    }
    case JAL: {
      int32_t imm21 = JumpOffset(instr);
      if (imm21 == kEndOfJumpChain) {
        // EndOfChain sentinel is returned directly, not relative to pc or pos.
        return kEndOfChain;
      } else {
        DEBUG_PRINTF("\t target_at: %d %d\n", imm21, pos.getOffset() + imm21);
        return pos.getOffset() + imm21;
      }
    }
    case JALR: {
      int32_t imm12 = instr >> 20;
      if (imm12 == kEndOfJumpChain) {
        // EndOfChain sentinel is returned directly, not relative to pc or pos.
        return kEndOfChain;
      } else {
        DEBUG_PRINTF("\t target_at: %d %d\n", imm12, pos.getOffset() + imm12);
        return pos.getOffset() + imm12;
      }
    }
    case LUI: {
      uintptr_t imm = target_address_at(instruction);
      uintptr_t instr_address = reinterpret_cast<uintptr_t>(instruction);
      if (imm == kEndOfJumpChain) {
        return kEndOfChain;
      } else {
        MOZ_ASSERT(instr_address - imm < INT_MAX);
        int32_t delta = static_cast<int32_t>(instr_address - imm);
        MOZ_ASSERT(pos.getOffset() > delta);
        return pos.getOffset() - delta;
      }
    }
    case AUIPC: {
      MOZ_ASSERT(instruction2 != nullptr);
      Instr instr_auipc = instr;
      Instr instr_I = instruction2->InstructionBits();
      MOZ_ASSERT(IsJalr(instr_I) || IsAddi(instr_I));
      int32_t offset = BrachlongOffset(instr_auipc, instr_I);
      if (offset == kEndOfJumpChain) return kEndOfChain;
      DEBUG_PRINTF("\t target_at: %d %d\n", offset, pos.getOffset() + offset);
      return offset + pos.getOffset();
    }
    default: {
      UNIMPLEMENTED_RISCV();
    }
  }
}

uint32_t Assembler::next_link(Label* L, bool is_internal) {
  MOZ_ASSERT(L->used());
  BufferOffset pos(L);
  int link = target_at(pos, is_internal);
  if (link == kEndOfChain) {
    L->reset();
    return LabelBase::INVALID_OFFSET;
  } else {
    MOZ_ASSERT(link >= 0);
    DEBUG_PRINTF("next: %p to offset %d\n", L, link);
    L->use(link);
    return link;
  }
}

void Assembler::bind(Label* label, BufferOffset boff) {
  JitSpew(JitSpew_Codegen, ".set Llabel %p %d", label, currentOffset());
  DEBUG_PRINTF(".set Llabel %p\n", label);
  // If our caller didn't give us an explicit target to bind to
  // then we want to bind to the location of the next instruction
  BufferOffset dest = boff.assigned() ? boff : nextOffset();
  if (label->used()) {
    uint32_t next;

    // A used label holds a link to branch that uses it.
    do {
      BufferOffset b(label);
      DEBUG_PRINTF("\tbind next:%d\n", b.getOffset());
      // Even a 0 offset may be invalid if we're out of memory.
      if (oom()) {
        return;
      }
      int fixup_pos = b.getOffset();
      int dist = dest.getOffset() - fixup_pos;
      next = next_link(label, false);
      DEBUG_PRINTF("\t%p fixup: %d next: %d\n", label, fixup_pos, next);
      DEBUG_PRINTF("\t   fixup: %d dest: %d dist: %d %d %d\n", fixup_pos,
                   dest.getOffset(), dist, nextOffset().getOffset(),
                   currentOffset());
      Instruction* instruction = editSrc(b);
      Instr instr = instruction->InstructionBits();
      if (IsBranch(instr)) {
        if (dist > kMaxBranchOffset) {
          MOZ_ASSERT(next != LabelBase::INVALID_OFFSET);
          MOZ_RELEASE_ASSERT((next - fixup_pos) <= kMaxBranchOffset);
          MOZ_ASSERT(IsAuipc(editSrc(BufferOffset(next))->InstructionBits()));
          MOZ_ASSERT(
              IsJalr(editSrc(BufferOffset(next + 4))->InstructionBits()));
          DEBUG_PRINTF("\t\ttrampolining: %d\n", next);
        } else {
          target_at_put(b, dest);
          BufferOffset deadline(b.getOffset() +
                                ImmBranchMaxForwardOffset(CondBranchRangeType));
          m_buffer.unregisterBranchDeadline(CondBranchRangeType, deadline);
        }
      } else if (IsJal(instr)) {
        if (dist > kMaxJumpOffset) {
          MOZ_ASSERT(next != LabelBase::INVALID_OFFSET);
          MOZ_RELEASE_ASSERT((next - fixup_pos) <= kMaxJumpOffset);
          MOZ_ASSERT(IsAuipc(editSrc(BufferOffset(next))->InstructionBits()));
          MOZ_ASSERT(
              IsJalr(editSrc(BufferOffset(next + 4))->InstructionBits()));
          DEBUG_PRINTF("\t\ttrampolining: %d\n", next);
        } else {
          target_at_put(b, dest);
          BufferOffset deadline(
              b.getOffset() + ImmBranchMaxForwardOffset(UncondBranchRangeType));
          m_buffer.unregisterBranchDeadline(UncondBranchRangeType, deadline);
        }
      } else {
        MOZ_ASSERT(IsAuipc(instr));
        target_at_put(b, dest);
      }
    } while (next != LabelBase::INVALID_OFFSET);
  }
  label->bind(dest.getOffset());
}

void Assembler::Bind(uint8_t* rawCode, const CodeLabel& label) {
  if (label.patchAt().bound()) {
    auto mode = label.linkMode();
    intptr_t offset = label.patchAt().offset();
    intptr_t target = label.target().offset();

    if (mode == CodeLabel::RawPointer) {
      *reinterpret_cast<const void**>(rawCode + offset) = rawCode + target;
    } else {
      MOZ_ASSERT(mode == CodeLabel::MoveImmediate ||
                 mode == CodeLabel::JumpImmediate);
      Instruction* inst = (Instruction*)(rawCode + offset);
      Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + target));
    }
  }
}

bool Assembler::is_near(Label* L) {
  MOZ_ASSERT(L->bound());
  return is_intn((currentOffset() - L->offset()), kJumpOffsetBits);
}

bool Assembler::is_near(Label* L, OffsetSize bits) {
  if (L == nullptr || !L->bound()) return true;
  return is_intn((currentOffset() - L->offset()), bits);
}

bool Assembler::is_near_branch(Label* L) {
  MOZ_ASSERT(L->bound());
  return is_intn((currentOffset() - L->offset()), kBranchOffsetBits);
}

int32_t Assembler::branch_long_offset(Label* L) {
  if (oom()) {
    return kEndOfJumpChain;
  }
  intptr_t target_pos;
  BufferOffset next_instr_offset = nextInstrOffset(2);
  DEBUG_PRINTF("\tbranch_long_offset: %p to (%d)\n", L,
               next_instr_offset.getOffset());
  if (L->bound()) {
    JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
            next_instr_offset.getOffset());
    target_pos = L->offset();
  } else {
    if (L->used()) {
      LabelCahe::Ptr p = label_cache_.lookup(L->offset());
      MOZ_ASSERT(p);
      MOZ_ASSERT(p->key() == L->offset());
      target_pos = p->value().getOffset();
      target_at_put(BufferOffset(target_pos), next_instr_offset);
      DEBUG_PRINTF("\tLabel  %p added to link: %d\n", L,
                   next_instr_offset.getOffset());
      bool ok = label_cache_.put(L->offset(), next_instr_offset);
      if (!ok) {
        NoEnoughLabelCache();
      }
      return kEndOfJumpChain;
    } else {
      JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
              next_instr_offset.getOffset());
      L->use(next_instr_offset.getOffset());
      DEBUG_PRINTF("\tLabel  %p added to link: %d\n", L,
                   next_instr_offset.getOffset());
      bool ok = label_cache_.putNew(L->offset(), next_instr_offset);
      if (!ok) {
        NoEnoughLabelCache();
      }
      return kEndOfJumpChain;
    }
  }
  intptr_t offset = target_pos - next_instr_offset.getOffset();
  MOZ_ASSERT((offset & 3) == 0);
  MOZ_ASSERT(is_int32(offset));
  return static_cast<int32_t>(offset);
}

int32_t Assembler::branch_offset_helper(Label* L, OffsetSize bits) {
  if (oom()) {
    return kEndOfJumpChain;
  }
  int32_t target_pos;
  BufferOffset next_instr_offset = nextInstrOffset();
  DEBUG_PRINTF("\tbranch_offset_helper: %p to %d\n", L,
               next_instr_offset.getOffset());
  // This is the last possible branch target.
  if (L->bound()) {
    JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
            next_instr_offset.getOffset());
    target_pos = L->offset();
  } else {
    BufferOffset deadline(next_instr_offset.getOffset() +
                          ImmBranchMaxForwardOffset(bits));
    DEBUG_PRINTF("\tregisterBranchDeadline %d type %d\n", deadline.getOffset(),
                 OffsetSizeToImmBranchRangeType(bits));
    m_buffer.registerBranchDeadline(OffsetSizeToImmBranchRangeType(bits),
                                    deadline);
    if (L->used()) {
      LabelCahe::Ptr p = label_cache_.lookup(L->offset());
      MOZ_ASSERT(p);
      MOZ_ASSERT(p->key() == L->offset());
      target_pos = p->value().getOffset();
      target_at_put(BufferOffset(target_pos), next_instr_offset);
      DEBUG_PRINTF("\tLabel  %p added to link: %d\n", L,
                   next_instr_offset.getOffset());
      bool ok = label_cache_.put(L->offset(), next_instr_offset);
      if (!ok) {
        NoEnoughLabelCache();
      }
      return kEndOfJumpChain;
    } else {
      JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
              next_instr_offset.getOffset());
      L->use(next_instr_offset.getOffset());
      bool ok = label_cache_.putNew(L->offset(), next_instr_offset);
      if (!ok) {
        NoEnoughLabelCache();
      }
      DEBUG_PRINTF("\tLabel  %p added to link: %d\n", L,
                   next_instr_offset.getOffset());
      return kEndOfJumpChain;
    }
  }

  int32_t offset = target_pos - next_instr_offset.getOffset();
  DEBUG_PRINTF("\toffset = %d\n", offset);
  MOZ_ASSERT(is_intn(offset, bits));
  MOZ_ASSERT((offset & 1) == 0);
  return offset;
}

Assembler::Condition Assembler::InvertCondition(Condition cond) {
  switch (cond) {
    case Equal:
      return NotEqual;
    case NotEqual:
      return Equal;
    case Zero:
      return NonZero;
    case NonZero:
      return Zero;
    case LessThan:
      return GreaterThanOrEqual;
    case LessThanOrEqual:
      return GreaterThan;
    case GreaterThan:
      return LessThanOrEqual;
    case GreaterThanOrEqual:
      return LessThan;
    case Above:
      return BelowOrEqual;
    case AboveOrEqual:
      return Below;
    case Below:
      return AboveOrEqual;
    case BelowOrEqual:
      return Above;
    case Signed:
      return NotSigned;
    case NotSigned:
      return Signed;
    default:
      MOZ_CRASH("unexpected condition");
  }
}

Assembler::DoubleCondition Assembler::InvertCondition(DoubleCondition cond) {
  switch (cond) {
    case DoubleOrdered:
      return DoubleUnordered;
    case DoubleEqual:
      return DoubleNotEqualOrUnordered;
    case DoubleNotEqual:
      return DoubleEqualOrUnordered;
    case DoubleGreaterThan:
      return DoubleLessThanOrEqualOrUnordered;
    case DoubleGreaterThanOrEqual:
      return DoubleLessThanOrUnordered;
    case DoubleLessThan:
      return DoubleGreaterThanOrEqualOrUnordered;
    case DoubleLessThanOrEqual:
      return DoubleGreaterThanOrUnordered;
    case DoubleUnordered:
      return DoubleOrdered;
    case DoubleEqualOrUnordered:
      return DoubleNotEqual;
    case DoubleNotEqualOrUnordered:
      return DoubleEqual;
    case DoubleGreaterThanOrUnordered:
      return DoubleLessThanOrEqual;
    case DoubleGreaterThanOrEqualOrUnordered:
      return DoubleLessThan;
    case DoubleLessThanOrUnordered:
      return DoubleGreaterThanOrEqual;
    case DoubleLessThanOrEqualOrUnordered:
      return DoubleGreaterThan;
    default:
      MOZ_CRASH("unexpected condition");
  }
}

// Break / Trap instructions.
void Assembler::break_(uint32_t code, bool break_as_stop) {
  // We need to invalidate breaks that could be stops as well because the
  // simulator expects a char pointer after the stop instruction.
  // See constants-mips.h for explanation.
  MOZ_ASSERT(
      (break_as_stop && code <= kMaxStopCode && code > kMaxTracepointCode) ||
      (!break_as_stop && (code > kMaxStopCode || code <= kMaxTracepointCode)));

  // since ebreak does not allow additional immediate field, we use the
  // immediate field of lui instruction immediately following the ebreak to
  // encode the "code" info
  ebreak();
  MOZ_ASSERT(is_uint20(code));
  lui(zero_reg, code);
}

void Assembler::ToggleToJmp(CodeLocationLabel inst_) {
  Instruction* inst = (Instruction*)inst_.raw();
  MOZ_ASSERT(IsAddi(inst->InstructionBits()));
  int32_t offset = inst->Imm12Value();
  MOZ_ASSERT(is_int12(offset));
  Instr jal_ = JAL | (0b000 << kFunct3Shift) |
               (offset & 0xff000) |          // bits 19-12
               ((offset & 0x800) << 9) |     // bit  11
               ((offset & 0x7fe) << 20) |    // bits 10-1
               ((offset & 0x100000) << 11);  // bit  20
  // jal(zero, offset);
  *reinterpret_cast<Instr*>(inst) = jal_;
}

void Assembler::ToggleToCmp(CodeLocationLabel inst_) {
  Instruction* inst = (Instruction*)inst_.raw();

  // toggledJump is allways used for short jumps.
  MOZ_ASSERT(IsJal(inst->InstructionBits()));
  // Replace "jal zero_reg, offset" with "addi $zero, $zero, offset"
  int32_t offset = inst->Imm20JValue();
  MOZ_ASSERT(is_int12(offset));
  Instr addi_ = OP_IMM | (0b000 << kFunct3Shift) |
                (offset << kImm12Shift);  // addi(zero, zero, low_12);
  *reinterpret_cast<Instr*>(inst) = addi_;
}

bool Assembler::reserve(size_t size) {
  // This buffer uses fixed-size chunks so there's no point in reserving
  // now vs. on-demand.
  return !oom();
}

static JitCode* CodeFromJump(Instruction* jump) {
  uint8_t* target = (uint8_t*)Assembler::ExtractLoad64Value(jump);
  return JitCode::FromExecutable(target);
}

void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
                                     CompactBufferReader& reader) {
  while (reader.more()) {
    JitCode* child =
        CodeFromJump((Instruction*)(code->raw() + reader.readUnsigned()));
    TraceManuallyBarrieredEdge(trc, &child, "rel32");
  }
}

static void TraceOneDataRelocation(JSTracer* trc,
                                   mozilla::Maybe<AutoWritableJitCode>& awjc,
                                   JitCode* code, Instruction* inst) {
  void* ptr = (void*)Assembler::ExtractLoad64Value(inst);
  void* prior = ptr;

  // Data relocations can be for Values or for raw pointers. If a Value is
  // zero-tagged, we can trace it as if it were a raw pointer. If a Value
  // is not zero-tagged, we have to interpret it as a Value to ensure that the
  // tag bits are masked off to recover the actual pointer.
  uintptr_t word = reinterpret_cast<uintptr_t>(ptr);
  if (word >> JSVAL_TAG_SHIFT) {
    // This relocation is a Value with a non-zero tag.
    Value v = Value::fromRawBits(word);
    TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value");
    ptr = (void*)v.bitsAsPunboxPointer();
  } else {
    // This relocation is a raw pointer or a Value with a zero tag.
    // No barrier needed since these are constants.
    TraceManuallyBarrieredGenericPointerEdge(
        trc, reinterpret_cast<gc::Cell**>(&ptr), "jit-masm-ptr");
  }

  if (ptr != prior) {
    if (awjc.isNothing()) {
      awjc.emplace(code);
    }
    Assembler::UpdateLoad64Value(inst, uint64_t(ptr));
  }
}

/* static */
void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code,
                                     CompactBufferReader& reader) {
  mozilla::Maybe<AutoWritableJitCode> awjc;
  while (reader.more()) {
    size_t offset = reader.readUnsigned();
    Instruction* inst = (Instruction*)(code->raw() + offset);
    TraceOneDataRelocation(trc, awjc, code, inst);
  }
}

UseScratchRegisterScope::UseScratchRegisterScope(Assembler* assembler)
    : available_(assembler->GetScratchRegisterList()),
      old_available_(*available_) {}

UseScratchRegisterScope::~UseScratchRegisterScope() {
  *available_ = old_available_;
}

Register UseScratchRegisterScope::Acquire() {
  MOZ_ASSERT(available_ != nullptr);
  MOZ_ASSERT(!available_->empty());
  Register index = GeneralRegisterSet::FirstRegister(available_->bits());
  available_->takeRegisterIndex(index);
  return index;
}

bool UseScratchRegisterScope::hasAvailable() const {
  return (available_->size()) != 0;
}

void Assembler::retarget(Label* label, Label* target) {
  spew("retarget %p -> %p", label, target);
  if (label->used() && !oom()) {
    if (target->bound()) {
      bind(label, BufferOffset(target));
    } else if (target->used()) {
      // The target is not bound but used. Prepend label's branch list
      // onto target's.
      int32_t next;
      BufferOffset labelBranchOffset(label);

      // Find the head of the use chain for label.
      do {
        next = next_link(label, false);
        labelBranchOffset = BufferOffset(next);
      } while (next != LabelBase::INVALID_OFFSET);

      // Then patch the head of label's use chain to the tail of
      // target's use chain, prepending the entire use chain of target.
      target->use(label->offset());
      target_at_put(labelBranchOffset, BufferOffset(target));
      MOZ_CRASH("check");
    } else {
      // The target is unbound and unused.  We can just take the head of
      // the list hanging off of label, and dump that into target.
      target->use(label->offset());
    }
  }
  label->reset();
}

bool Assembler::appendRawCode(const uint8_t* code, size_t numBytes) {
  if (m_buffer.oom()) {
    return false;
  }
  while (numBytes > SliceSize) {
    m_buffer.putBytes(SliceSize, code);
    numBytes -= SliceSize;
    code += SliceSize;
  }
  m_buffer.putBytes(numBytes, code);
  return !m_buffer.oom();
}

void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) {
  Instruction* i0 = (Instruction*)inst_.raw();
  Instruction* i1 = (Instruction*)(inst_.raw() + 1 * kInstrSize);
  Instruction* i2 = (Instruction*)(inst_.raw() + 2 * kInstrSize);
  Instruction* i3 = (Instruction*)(inst_.raw() + 3 * kInstrSize);
  Instruction* i4 = (Instruction*)(inst_.raw() + 4 * kInstrSize);
  Instruction* i5 = (Instruction*)(inst_.raw() + 5 * kInstrSize);
  Instruction* i6 = (Instruction*)(inst_.raw() + 6 * kInstrSize);

  MOZ_ASSERT(IsLui(i0->InstructionBits()));
  MOZ_ASSERT(IsAddi(i1->InstructionBits()));
  MOZ_ASSERT(IsSlli(i2->InstructionBits()));
  MOZ_ASSERT(IsOri(i3->InstructionBits()));
  MOZ_ASSERT(IsSlli(i4->InstructionBits()));
  MOZ_ASSERT(IsOri(i5->InstructionBits()));
  if (enabled) {
    Instr jalr_ = JALR | (ra.code() << kRdShift) | (0x0 << kFunct3Shift) |
                  (i5->RdValue() << kRs1Shift) | (0x0 << kImm12Shift);
    *((Instr*)i6) = jalr_;
  } else {
    *((Instr*)i6) = kNopByte;
  }
}

void Assembler::PatchShortRangeBranchToVeneer(Buffer* buffer, unsigned rangeIdx,
                                              BufferOffset deadline,
                                              BufferOffset veneer) {
  if (buffer->oom()) {
    return;
  }
  DEBUG_PRINTF("\tPatchShortRangeBranchToVeneer\n");
  // Reconstruct the position of the branch from (rangeIdx, deadline).
  ImmBranchRangeType branchRange = static_cast<ImmBranchRangeType>(rangeIdx);
  BufferOffset branch(deadline.getOffset() -
                      ImmBranchMaxForwardOffset(branchRange));
  Instruction* branchInst = buffer->getInst(branch);
  Instruction* veneerInst_1 = buffer->getInst(veneer);
  Instruction* veneerInst_2 =
      buffer->getInst(BufferOffset(veneer.getOffset() + 4));
  // Verify that the branch range matches what's encoded.
  DEBUG_PRINTF("\t%p(%x): ", branchInst, branch.getOffset());
  disassembleInstr(branchInst->InstructionBits(), JitSpew_Codegen);
  DEBUG_PRINTF("\t instert veneer %x, branch:%x deadline: %x\n",
               veneer.getOffset(), branch.getOffset(), deadline.getOffset());
  MOZ_ASSERT(branchRange <= UncondBranchRangeType);
  MOZ_ASSERT(branchInst->GetImmBranchRangeType() == branchRange);
  // emit a long jump slot
  Instr auipc = AUIPC | (t6.code() << kRdShift) | (0x0 << kImm20Shift);
  Instr jalr = JALR | (zero_reg.code() << kRdShift) | (0x0 << kFunct3Shift) |
               (t6.code() << kRs1Shift) | (0x0 << kImm12Shift);

  // We want to insert veneer after branch in the linked list of instructions
  // that use the same unbound label.
  // The veneer should be an unconditional branch.
  int32_t nextElemOffset = target_at(buffer->getInst(branch), branch, false);
  int32_t dist;
  // If offset is 0, this is the end of the linked list.
  if (nextElemOffset != kEndOfChain) {
    // Make the offset relative to veneer so it targets the same instruction
    // as branchInst.
    dist = nextElemOffset - veneer.getOffset();
  } else {
    dist = 0;
  }
  int32_t Hi20 = (((int32_t)dist + 0x800) >> 12);
  int32_t Lo12 = (int32_t)dist << 20 >> 20;
  auipc = SetAuipcOffset(Hi20, auipc);
  jalr = SetJalrOffset(Lo12, jalr);
  // insert veneer
  veneerInst_1->SetInstructionBits(auipc);
  veneerInst_2->SetInstructionBits(jalr);
  // Now link branchInst to veneer.
  if (IsBranch(branchInst->InstructionBits())) {
    branchInst->SetInstructionBits(SetBranchOffset(
        branch.getOffset(), veneer.getOffset(), branchInst->InstructionBits()));
  } else {
    MOZ_ASSERT(IsJal(branchInst->InstructionBits()));
    branchInst->SetInstructionBits(SetJalOffset(
        branch.getOffset(), veneer.getOffset(), branchInst->InstructionBits()));
  }
  DEBUG_PRINTF("\tfix to veneer:");
  disassembleInstr(branchInst->InstructionBits());
}
}  // namespace jit
}  // namespace js