summaryrefslogtreecommitdiffstats
path: root/js/src/jit/riscv64/Assembler-riscv64.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/jit/riscv64/Assembler-riscv64.cpp')
-rw-r--r--js/src/jit/riscv64/Assembler-riscv64.cpp1548
1 files changed, 1548 insertions, 0 deletions
diff --git a/js/src/jit/riscv64/Assembler-riscv64.cpp b/js/src/jit/riscv64/Assembler-riscv64.cpp
new file mode 100644
index 0000000000..5370192987
--- /dev/null
+++ b/js/src/jit/riscv64/Assembler-riscv64.cpp
@@ -0,0 +1,1548 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Copyright (c) 1994-2006 Sun Microsystems Inc.
+// All Rights Reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// - Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+//
+// - Redistribution in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// - Neither the name of Sun Microsystems or the names of contributors may
+// be used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The original source code covered by the above license above has been
+// modified significantly by Google Inc.
+// Copyright 2021 the V8 project authors. All rights reserved.
+#include "jit/riscv64/Assembler-riscv64.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Maybe.h"
+
+#include "gc/Marking.h"
+#include "jit/AutoWritableJitCode.h"
+#include "jit/ExecutableAllocator.h"
+#include "jit/riscv64/disasm/Disasm-riscv64.h"
+#include "vm/Realm.h"
+
+using mozilla::DebugOnly;
+namespace js {
+namespace jit {
+
+#define UNIMPLEMENTED_RISCV() MOZ_CRASH("RISC_V not implemented");
+
+bool Assembler::FLAG_riscv_debug = false;
+
+void Assembler::nop() { addi(ToRegister(0), ToRegister(0), 0); }
+
+// Size of the instruction stream, in bytes.
+size_t Assembler::size() const { return m_buffer.size(); }
+
+bool Assembler::swapBuffer(wasm::Bytes& bytes) {
+ // For now, specialize to the one use case. As long as wasm::Bytes is a
+ // Vector, not a linked-list of chunks, there's not much we can do other
+ // than copy.
+ MOZ_ASSERT(bytes.empty());
+ if (!bytes.resize(bytesNeeded())) {
+ return false;
+ }
+ m_buffer.executableCopy(bytes.begin());
+ return true;
+}
+
+// Size of the relocation table, in bytes.
+size_t Assembler::jumpRelocationTableBytes() const {
+ return jumpRelocations_.length();
+}
+
+size_t Assembler::dataRelocationTableBytes() const {
+ return dataRelocations_.length();
+}
+// Size of the data table, in bytes.
+size_t Assembler::bytesNeeded() const {
+ return size() + jumpRelocationTableBytes() + dataRelocationTableBytes();
+}
+
+void Assembler::executableCopy(uint8_t* buffer) {
+ MOZ_ASSERT(isFinished);
+ m_buffer.executableCopy(buffer);
+}
+
+uint32_t Assembler::AsmPoolMaxOffset = 1024;
+
+uint32_t Assembler::GetPoolMaxOffset() {
+ static bool isSet = false;
+ if (!isSet) {
+ char* poolMaxOffsetStr = getenv("ASM_POOL_MAX_OFFSET");
+ uint32_t poolMaxOffset;
+ if (poolMaxOffsetStr &&
+ sscanf(poolMaxOffsetStr, "%u", &poolMaxOffset) == 1) {
+ AsmPoolMaxOffset = poolMaxOffset;
+ }
+ isSet = true;
+ }
+ return AsmPoolMaxOffset;
+}
+
+// Pool callbacks stuff:
+void Assembler::InsertIndexIntoTag(uint8_t* load_, uint32_t index) {
+ MOZ_CRASH("Unimplement");
+}
+
+void Assembler::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr) {
+ MOZ_CRASH("Unimplement");
+}
+
+void Assembler::processCodeLabels(uint8_t* rawCode) {
+ for (const CodeLabel& label : codeLabels_) {
+ Bind(rawCode, label);
+ }
+}
+
+void Assembler::WritePoolGuard(BufferOffset branch, Instruction* dest,
+ BufferOffset afterPool) {
+ DEBUG_PRINTF("\tWritePoolGuard\n");
+ int32_t off = afterPool.getOffset() - branch.getOffset();
+ if (!is_int21(off) || !((off & 0x1) == 0)) {
+ printf("%d\n", off);
+ MOZ_CRASH("imm invalid");
+ }
+ // JAL encode is
+ // 31 | 30 21 | 20 | 19 12 | 11 7 | 6 0 |
+ // imm[20] | imm[10:1] | imm[11] | imm[19:12] | rd | opcode|
+ // 1 10 1 8 5 7
+ // offset[20:1] dest JAL
+ int32_t imm20 = (off & 0xff000) | // bits 19-12
+ ((off & 0x800) << 9) | // bit 11
+ ((off & 0x7fe) << 20) | // bits 10-1
+ ((off & 0x100000) << 11); // bit 20
+ Instr instr = JAL | (imm20 & kImm20Mask);
+ dest->SetInstructionBits(instr);
+ DEBUG_PRINTF("%p(%x): ", dest, branch.getOffset());
+ disassembleInstr(dest->InstructionBits(), JitSpew_Codegen);
+}
+
+void Assembler::WritePoolHeader(uint8_t* start, Pool* p, bool isNatural) {
+ static_assert(sizeof(PoolHeader) == 4);
+
+ // Get the total size of the pool.
+ const uintptr_t totalPoolSize = sizeof(PoolHeader) + p->getPoolSize();
+ const uintptr_t totalPoolInstructions = totalPoolSize / kInstrSize;
+
+ MOZ_ASSERT((totalPoolSize & 0x3) == 0);
+ MOZ_ASSERT(totalPoolInstructions < (1 << 15));
+
+ PoolHeader header(totalPoolInstructions, isNatural);
+ *(PoolHeader*)start = header;
+}
+
+void Assembler::copyJumpRelocationTable(uint8_t* dest) {
+ if (jumpRelocations_.length()) {
+ memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
+ }
+}
+
+void Assembler::copyDataRelocationTable(uint8_t* dest) {
+ if (dataRelocations_.length()) {
+ memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
+ }
+}
+
+void Assembler::RV_li(Register rd, int64_t imm) {
+ UseScratchRegisterScope temps(this);
+ if (RecursiveLiCount(imm) > GeneralLiCount(imm, temps.hasAvailable())) {
+ GeneralLi(rd, imm);
+ } else {
+ RecursiveLi(rd, imm);
+ }
+}
+
+int Assembler::RV_li_count(int64_t imm, bool is_get_temp_reg) {
+ if (RecursiveLiCount(imm) > GeneralLiCount(imm, is_get_temp_reg)) {
+ return GeneralLiCount(imm, is_get_temp_reg);
+ } else {
+ return RecursiveLiCount(imm);
+ }
+}
+
+void Assembler::GeneralLi(Register rd, int64_t imm) {
+ // 64-bit imm is put in the register rd.
+ // In most cases the imm is 32 bit and 2 instructions are generated. If a
+ // temporary register is available, in the worst case, 6 instructions are
+ // generated for a full 64-bit immediate. If temporay register is not
+ // available the maximum will be 8 instructions. If imm is more than 32 bits
+ // and a temp register is available, imm is divided into two 32-bit parts,
+ // low_32 and up_32. Each part is built in a separate register. low_32 is
+ // built before up_32. If low_32 is negative (upper 32 bits are 1), 0xffffffff
+ // is subtracted from up_32 before up_32 is built. This compensates for 32
+ // bits of 1's in the lower when the two registers are added. If no temp is
+ // available, the upper 32 bit is built in rd, and the lower 32 bits are
+ // devided to 3 parts (11, 11, and 10 bits). The parts are shifted and added
+ // to the upper part built in rd.
+ if (is_int32(imm + 0x800)) {
+ // 32-bit case. Maximum of 2 instructions generated
+ int64_t high_20 = ((imm + 0x800) >> 12);
+ int64_t low_12 = imm << 52 >> 52;
+ if (high_20) {
+ lui(rd, (int32_t)high_20);
+ if (low_12) {
+ addi(rd, rd, low_12);
+ }
+ } else {
+ addi(rd, zero_reg, low_12);
+ }
+ return;
+ } else {
+ UseScratchRegisterScope temps(this);
+ BlockTrampolinePoolScope block_trampoline_pool(this, 8);
+ // 64-bit case: divide imm into two 32-bit parts, upper and lower
+ int64_t up_32 = imm >> 32;
+ int64_t low_32 = imm & 0xffffffffull;
+ Register temp_reg = rd;
+ // Check if a temporary register is available
+ if (up_32 == 0 || low_32 == 0) {
+ // No temp register is needed
+ } else {
+ temp_reg = temps.hasAvailable() ? temps.Acquire() : InvalidReg;
+ }
+ if (temp_reg != InvalidReg) {
+ // keep track of hardware behavior for lower part in sim_low
+ int64_t sim_low = 0;
+ // Build lower part
+ if (low_32 != 0) {
+ int64_t high_20 = ((low_32 + 0x800) >> 12);
+ int64_t low_12 = low_32 & 0xfff;
+ if (high_20) {
+ // Adjust to 20 bits for the case of overflow
+ high_20 &= 0xfffff;
+ sim_low = ((high_20 << 12) << 32) >> 32;
+ lui(rd, (int32_t)high_20);
+ if (low_12) {
+ sim_low += (low_12 << 52 >> 52) | low_12;
+ addi(rd, rd, low_12);
+ }
+ } else {
+ sim_low = low_12;
+ ori(rd, zero_reg, low_12);
+ }
+ }
+ if (sim_low & 0x100000000) {
+ // Bit 31 is 1. Either an overflow or a negative 64 bit
+ if (up_32 == 0) {
+ // Positive number, but overflow because of the add 0x800
+ slli(rd, rd, 32);
+ srli(rd, rd, 32);
+ return;
+ }
+ // low_32 is a negative 64 bit after the build
+ up_32 = (up_32 - 0xffffffff) & 0xffffffff;
+ }
+ if (up_32 == 0) {
+ return;
+ }
+ // Build upper part in a temporary register
+ if (low_32 == 0) {
+ // Build upper part in rd
+ temp_reg = rd;
+ }
+ int64_t high_20 = (up_32 + 0x800) >> 12;
+ int64_t low_12 = up_32 & 0xfff;
+ if (high_20) {
+ // Adjust to 20 bits for the case of overflow
+ high_20 &= 0xfffff;
+ lui(temp_reg, (int32_t)high_20);
+ if (low_12) {
+ addi(temp_reg, temp_reg, low_12);
+ }
+ } else {
+ ori(temp_reg, zero_reg, low_12);
+ }
+ // Put it at the bgining of register
+ slli(temp_reg, temp_reg, 32);
+ if (low_32 != 0) {
+ add(rd, rd, temp_reg);
+ }
+ return;
+ }
+ // No temp register. Build imm in rd.
+ // Build upper 32 bits first in rd. Divide lower 32 bits parts and add
+ // parts to the upper part by doing shift and add.
+ // First build upper part in rd.
+ int64_t high_20 = (up_32 + 0x800) >> 12;
+ int64_t low_12 = up_32 & 0xfff;
+ if (high_20) {
+ // Adjust to 20 bits for the case of overflow
+ high_20 &= 0xfffff;
+ lui(rd, (int32_t)high_20);
+ if (low_12) {
+ addi(rd, rd, low_12);
+ }
+ } else {
+ ori(rd, zero_reg, low_12);
+ }
+ // upper part already in rd. Each part to be added to rd, has maximum of 11
+ // bits, and always starts with a 1. rd is shifted by the size of the part
+ // plus the number of zeros between the parts. Each part is added after the
+ // left shift.
+ uint32_t mask = 0x80000000;
+ int32_t shift_val = 0;
+ int32_t i;
+ for (i = 0; i < 32; i++) {
+ if ((low_32 & mask) == 0) {
+ mask >>= 1;
+ shift_val++;
+ if (i == 31) {
+ // rest is zero
+ slli(rd, rd, shift_val);
+ }
+ continue;
+ }
+ // The first 1 seen
+ int32_t part;
+ if ((i + 11) < 32) {
+ // Pick 11 bits
+ part = ((uint32_t)(low_32 << i) >> i) >> (32 - (i + 11));
+ slli(rd, rd, shift_val + 11);
+ ori(rd, rd, part);
+ i += 10;
+ mask >>= 11;
+ } else {
+ part = (uint32_t)(low_32 << i) >> i;
+ slli(rd, rd, shift_val + (32 - i));
+ ori(rd, rd, part);
+ break;
+ }
+ shift_val = 0;
+ }
+ }
+}
+
+int Assembler::GeneralLiCount(int64_t imm, bool is_get_temp_reg) {
+ int count = 0;
+ // imitate Assembler::RV_li
+ if (is_int32(imm + 0x800)) {
+ // 32-bit case. Maximum of 2 instructions generated
+ int64_t high_20 = ((imm + 0x800) >> 12);
+ int64_t low_12 = imm << 52 >> 52;
+ if (high_20) {
+ count++;
+ if (low_12) {
+ count++;
+ }
+ } else {
+ count++;
+ }
+ return count;
+ } else {
+ // 64-bit case: divide imm into two 32-bit parts, upper and lower
+ int64_t up_32 = imm >> 32;
+ int64_t low_32 = imm & 0xffffffffull;
+ // Check if a temporary register is available
+ if (is_get_temp_reg) {
+ // keep track of hardware behavior for lower part in sim_low
+ int64_t sim_low = 0;
+ // Build lower part
+ if (low_32 != 0) {
+ int64_t high_20 = ((low_32 + 0x800) >> 12);
+ int64_t low_12 = low_32 & 0xfff;
+ if (high_20) {
+ // Adjust to 20 bits for the case of overflow
+ high_20 &= 0xfffff;
+ sim_low = ((high_20 << 12) << 32) >> 32;
+ count++;
+ if (low_12) {
+ sim_low += (low_12 << 52 >> 52) | low_12;
+ count++;
+ }
+ } else {
+ sim_low = low_12;
+ count++;
+ }
+ }
+ if (sim_low & 0x100000000) {
+ // Bit 31 is 1. Either an overflow or a negative 64 bit
+ if (up_32 == 0) {
+ // Positive number, but overflow because of the add 0x800
+ count++;
+ count++;
+ return count;
+ }
+ // low_32 is a negative 64 bit after the build
+ up_32 = (up_32 - 0xffffffff) & 0xffffffff;
+ }
+ if (up_32 == 0) {
+ return count;
+ }
+ int64_t high_20 = (up_32 + 0x800) >> 12;
+ int64_t low_12 = up_32 & 0xfff;
+ if (high_20) {
+ // Adjust to 20 bits for the case of overflow
+ high_20 &= 0xfffff;
+ count++;
+ if (low_12) {
+ count++;
+ }
+ } else {
+ count++;
+ }
+ // Put it at the bgining of register
+ count++;
+ if (low_32 != 0) {
+ count++;
+ }
+ return count;
+ }
+ // No temp register. Build imm in rd.
+ // Build upper 32 bits first in rd. Divide lower 32 bits parts and add
+ // parts to the upper part by doing shift and add.
+ // First build upper part in rd.
+ int64_t high_20 = (up_32 + 0x800) >> 12;
+ int64_t low_12 = up_32 & 0xfff;
+ if (high_20) {
+ // Adjust to 20 bits for the case of overflow
+ high_20 &= 0xfffff;
+ count++;
+ if (low_12) {
+ count++;
+ }
+ } else {
+ count++;
+ }
+ // upper part already in rd. Each part to be added to rd, has maximum of 11
+ // bits, and always starts with a 1. rd is shifted by the size of the part
+ // plus the number of zeros between the parts. Each part is added after the
+ // left shift.
+ uint32_t mask = 0x80000000;
+ int32_t i;
+ for (i = 0; i < 32; i++) {
+ if ((low_32 & mask) == 0) {
+ mask >>= 1;
+ if (i == 31) {
+ // rest is zero
+ count++;
+ }
+ continue;
+ }
+ // The first 1 seen
+ if ((i + 11) < 32) {
+ // Pick 11 bits
+ count++;
+ count++;
+ i += 10;
+ mask >>= 11;
+ } else {
+ count++;
+ count++;
+ break;
+ }
+ }
+ }
+ return count;
+}
+
+void Assembler::li_ptr(Register rd, int64_t imm) {
+ m_buffer.enterNoNops();
+ m_buffer.assertNoPoolAndNoNops();
+ // Initialize rd with an address
+ // Pointers are 48 bits
+ // 6 fixed instructions are generated
+ DEBUG_PRINTF("li_ptr(%d, %lx <%ld>)\n", ToNumber(rd), imm, imm);
+ MOZ_ASSERT((imm & 0xfff0000000000000ll) == 0);
+ int64_t a6 = imm & 0x3f; // bits 0:5. 6 bits
+ int64_t b11 = (imm >> 6) & 0x7ff; // bits 6:11. 11 bits
+ int64_t high_31 = (imm >> 17) & 0x7fffffff; // 31 bits
+ int64_t high_20 = ((high_31 + 0x800) >> 12); // 19 bits
+ int64_t low_12 = high_31 & 0xfff; // 12 bits
+ lui(rd, (int32_t)high_20);
+ addi(rd, rd, low_12); // 31 bits in rd.
+ slli(rd, rd, 11); // Space for next 11 bis
+ ori(rd, rd, b11); // 11 bits are put in. 42 bit in rd
+ slli(rd, rd, 6); // Space for next 6 bits
+ ori(rd, rd, a6); // 6 bits are put in. 48 bis in rd
+ m_buffer.leaveNoNops();
+}
+
+void Assembler::li_constant(Register rd, int64_t imm) {
+ m_buffer.enterNoNops();
+ m_buffer.assertNoPoolAndNoNops();
+ DEBUG_PRINTF("li_constant(%d, %lx <%ld>)\n", ToNumber(rd), imm, imm);
+ lui(rd, (imm + (1LL << 47) + (1LL << 35) + (1LL << 23) + (1LL << 11)) >>
+ 48); // Bits 63:48
+ addiw(rd, rd,
+ (imm + (1LL << 35) + (1LL << 23) + (1LL << 11)) << 16 >>
+ 52); // Bits 47:36
+ slli(rd, rd, 12);
+ addi(rd, rd, (imm + (1LL << 23) + (1LL << 11)) << 28 >> 52); // Bits 35:24
+ slli(rd, rd, 12);
+ addi(rd, rd, (imm + (1LL << 11)) << 40 >> 52); // Bits 23:12
+ slli(rd, rd, 12);
+ addi(rd, rd, imm << 52 >> 52); // Bits 11:0
+ m_buffer.leaveNoNops();
+}
+
+ABIArg ABIArgGenerator::next(MIRType type) {
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Int64:
+ case MIRType::Pointer:
+ case MIRType::WasmAnyRef:
+ case MIRType::StackResults: {
+ if (intRegIndex_ == NumIntArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uintptr_t);
+ break;
+ }
+ current_ = ABIArg(Register::FromCode(intRegIndex_ + a0.encoding()));
+ intRegIndex_++;
+ break;
+ }
+ case MIRType::Float32:
+ case MIRType::Double: {
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(double);
+ break;
+ }
+ current_ = ABIArg(FloatRegister(
+ FloatRegisters::Encoding(floatRegIndex_ + fa0.encoding()),
+ type == MIRType::Double ? FloatRegisters::Double
+ : FloatRegisters::Single));
+ floatRegIndex_++;
+ break;
+ }
+ case MIRType::Simd128: {
+ MOZ_CRASH("RISCV64 does not support simd yet.");
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+}
+
+bool Assembler::oom() const {
+ return AssemblerShared::oom() || m_buffer.oom() || jumpRelocations_.oom() ||
+ dataRelocations_.oom() || !enoughLabelCache_;
+}
+
+int Assembler::disassembleInstr(Instr instr, bool enable_spew) {
+ if (!FLAG_riscv_debug && !enable_spew) return -1;
+ disasm::NameConverter converter;
+ disasm::Disassembler disasm(converter);
+ EmbeddedVector<char, 128> disasm_buffer;
+
+ int size =
+ disasm.InstructionDecode(disasm_buffer, reinterpret_cast<byte*>(&instr));
+ DEBUG_PRINTF("%s\n", disasm_buffer.start());
+ if (enable_spew) {
+ JitSpew(JitSpew_Codegen, "%s", disasm_buffer.start());
+ }
+ return size;
+}
+
+uintptr_t Assembler::target_address_at(Instruction* pc) {
+ Instruction* instr0 = pc;
+ DEBUG_PRINTF("target_address_at: pc: 0x%p\t", instr0);
+ Instruction* instr1 = pc + 1 * kInstrSize;
+ Instruction* instr2 = pc + 2 * kInstrSize;
+ Instruction* instr3 = pc + 3 * kInstrSize;
+ Instruction* instr4 = pc + 4 * kInstrSize;
+ Instruction* instr5 = pc + 5 * kInstrSize;
+
+ // Interpret instructions for address generated by li: See listing in
+ // Assembler::set_target_address_at() just below.
+ if (IsLui(*reinterpret_cast<Instr*>(instr0)) &&
+ IsAddi(*reinterpret_cast<Instr*>(instr1)) &&
+ IsSlli(*reinterpret_cast<Instr*>(instr2)) &&
+ IsOri(*reinterpret_cast<Instr*>(instr3)) &&
+ IsSlli(*reinterpret_cast<Instr*>(instr4)) &&
+ IsOri(*reinterpret_cast<Instr*>(instr5))) {
+ // Assemble the 64 bit value.
+ int64_t addr = (int64_t)(instr0->Imm20UValue() << kImm20Shift) +
+ (int64_t)instr1->Imm12Value();
+ MOZ_ASSERT(instr2->Imm12Value() == 11);
+ addr <<= 11;
+ addr |= (int64_t)instr3->Imm12Value();
+ MOZ_ASSERT(instr4->Imm12Value() == 6);
+ addr <<= 6;
+ addr |= (int64_t)instr5->Imm12Value();
+
+ DEBUG_PRINTF("addr: %lx\n", addr);
+ return static_cast<uintptr_t>(addr);
+ }
+ // We should never get here, force a bad address if we do.
+ MOZ_CRASH("RISC-V UNREACHABLE");
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ ImmPtr newValue, ImmPtr expectedValue) {
+ PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value),
+ PatchedImmPtr(expectedValue.value));
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue) {
+ Instruction* inst = (Instruction*)label.raw();
+
+ // Extract old Value
+ DebugOnly<uint64_t> value = Assembler::ExtractLoad64Value(inst);
+ MOZ_ASSERT(value == uint64_t(expectedValue.value));
+
+ // Replace with new value
+ Assembler::UpdateLoad64Value(inst, uint64_t(newValue.value));
+}
+
+uint64_t Assembler::ExtractLoad64Value(Instruction* inst0) {
+ DEBUG_PRINTF("\tExtractLoad64Value: \tpc:%p ", inst0);
+ if (IsJal(*reinterpret_cast<Instr*>(inst0))) {
+ int offset = inst0->Imm20JValue();
+ inst0 = inst0 + offset;
+ }
+ Instruction* instr1 = inst0 + 1 * kInstrSize;
+ if (IsAddiw(*reinterpret_cast<Instr*>(instr1))) {
+ // Li64
+ Instruction* instr2 = inst0 + 2 * kInstrSize;
+ Instruction* instr3 = inst0 + 3 * kInstrSize;
+ Instruction* instr4 = inst0 + 4 * kInstrSize;
+ Instruction* instr5 = inst0 + 5 * kInstrSize;
+ Instruction* instr6 = inst0 + 6 * kInstrSize;
+ Instruction* instr7 = inst0 + 7 * kInstrSize;
+ if (IsLui(*reinterpret_cast<Instr*>(inst0)) &&
+ IsAddiw(*reinterpret_cast<Instr*>(instr1)) &&
+ IsSlli(*reinterpret_cast<Instr*>(instr2)) &&
+ IsAddi(*reinterpret_cast<Instr*>(instr3)) &&
+ IsSlli(*reinterpret_cast<Instr*>(instr4)) &&
+ IsAddi(*reinterpret_cast<Instr*>(instr5)) &&
+ IsSlli(*reinterpret_cast<Instr*>(instr6)) &&
+ IsAddi(*reinterpret_cast<Instr*>(instr7))) {
+ int64_t imm = (int64_t)(inst0->Imm20UValue() << kImm20Shift) +
+ (int64_t)instr1->Imm12Value();
+ MOZ_ASSERT(instr2->Imm12Value() == 12);
+ imm <<= 12;
+ imm += (int64_t)instr3->Imm12Value();
+ MOZ_ASSERT(instr4->Imm12Value() == 12);
+ imm <<= 12;
+ imm += (int64_t)instr5->Imm12Value();
+ MOZ_ASSERT(instr6->Imm12Value() == 12);
+ imm <<= 12;
+ imm += (int64_t)instr7->Imm12Value();
+ DEBUG_PRINTF("imm:%lx\n", imm);
+ return imm;
+ } else {
+ FLAG_riscv_debug = true;
+ disassembleInstr(inst0->InstructionBits());
+ disassembleInstr(instr1->InstructionBits());
+ disassembleInstr(instr2->InstructionBits());
+ disassembleInstr(instr3->InstructionBits());
+ disassembleInstr(instr4->InstructionBits());
+ disassembleInstr(instr5->InstructionBits());
+ disassembleInstr(instr6->InstructionBits());
+ disassembleInstr(instr7->InstructionBits());
+ MOZ_CRASH();
+ }
+ } else {
+ DEBUG_PRINTF("\n");
+ Instruction* instrf1 = (inst0 - 1 * kInstrSize);
+ Instruction* instr2 = inst0 + 2 * kInstrSize;
+ Instruction* instr3 = inst0 + 3 * kInstrSize;
+ Instruction* instr4 = inst0 + 4 * kInstrSize;
+ Instruction* instr5 = inst0 + 5 * kInstrSize;
+ Instruction* instr6 = inst0 + 6 * kInstrSize;
+ Instruction* instr7 = inst0 + 7 * kInstrSize;
+ disassembleInstr(instrf1->InstructionBits());
+ disassembleInstr(inst0->InstructionBits());
+ disassembleInstr(instr1->InstructionBits());
+ disassembleInstr(instr2->InstructionBits());
+ disassembleInstr(instr3->InstructionBits());
+ disassembleInstr(instr4->InstructionBits());
+ disassembleInstr(instr5->InstructionBits());
+ disassembleInstr(instr6->InstructionBits());
+ disassembleInstr(instr7->InstructionBits());
+ MOZ_ASSERT(IsAddi(*reinterpret_cast<Instr*>(instr1)));
+ // Li48
+ return target_address_at(inst0);
+ }
+}
+
+void Assembler::UpdateLoad64Value(Instruction* pc, uint64_t value) {
+ DEBUG_PRINTF("\tUpdateLoad64Value: pc: %p\tvalue: %lx\n", pc, value);
+ Instruction* instr1 = pc + 1 * kInstrSize;
+ if (IsJal(*reinterpret_cast<Instr*>(pc))) {
+ pc = pc + pc->Imm20JValue();
+ instr1 = pc + 1 * kInstrSize;
+ }
+ if (IsAddiw(*reinterpret_cast<Instr*>(instr1))) {
+ Instruction* instr0 = pc;
+ Instruction* instr2 = pc + 2 * kInstrSize;
+ Instruction* instr3 = pc + 3 * kInstrSize;
+ Instruction* instr4 = pc + 4 * kInstrSize;
+ Instruction* instr5 = pc + 5 * kInstrSize;
+ Instruction* instr6 = pc + 6 * kInstrSize;
+ Instruction* instr7 = pc + 7 * kInstrSize;
+ MOZ_ASSERT(IsLui(*reinterpret_cast<Instr*>(pc)) &&
+ IsAddiw(*reinterpret_cast<Instr*>(instr1)) &&
+ IsSlli(*reinterpret_cast<Instr*>(instr2)) &&
+ IsAddi(*reinterpret_cast<Instr*>(instr3)) &&
+ IsSlli(*reinterpret_cast<Instr*>(instr4)) &&
+ IsAddi(*reinterpret_cast<Instr*>(instr5)) &&
+ IsSlli(*reinterpret_cast<Instr*>(instr6)) &&
+ IsAddi(*reinterpret_cast<Instr*>(instr7)));
+ // lui(rd, (imm + (1LL << 47) + (1LL << 35) + (1LL << 23) + (1LL << 11)) >>
+ // 48); // Bits 63:48
+ // addiw(rd, rd,
+ // (imm + (1LL << 35) + (1LL << 23) + (1LL << 11)) << 16 >>
+ // 52); // Bits 47:36
+ // slli(rd, rd, 12);
+ // addi(rd, rd, (imm + (1LL << 23) + (1LL << 11)) << 28 >> 52); // Bits
+ // 35:24 slli(rd, rd, 12); addi(rd, rd, (imm + (1LL << 11)) << 40 >> 52); //
+ // Bits 23:12 slli(rd, rd, 12); addi(rd, rd, imm << 52 >> 52); // Bits 11:0
+ *reinterpret_cast<Instr*>(instr0) &= 0xfff;
+ *reinterpret_cast<Instr*>(instr0) |=
+ (((value + (1LL << 47) + (1LL << 35) + (1LL << 23) + (1LL << 11)) >> 48)
+ << 12);
+ *reinterpret_cast<Instr*>(instr1) &= 0xfffff;
+ *reinterpret_cast<Instr*>(instr1) |=
+ (((value + (1LL << 35) + (1LL << 23) + (1LL << 11)) << 16 >> 52) << 20);
+ *reinterpret_cast<Instr*>(instr3) &= 0xfffff;
+ *reinterpret_cast<Instr*>(instr3) |=
+ (((value + (1LL << 23) + (1LL << 11)) << 28 >> 52) << 20);
+ *reinterpret_cast<Instr*>(instr5) &= 0xfffff;
+ *reinterpret_cast<Instr*>(instr5) |=
+ (((value + (1LL << 11)) << 40 >> 52) << 20);
+ *reinterpret_cast<Instr*>(instr7) &= 0xfffff;
+ *reinterpret_cast<Instr*>(instr7) |= ((value << 52 >> 52) << 20);
+ disassembleInstr(instr0->InstructionBits());
+ disassembleInstr(instr1->InstructionBits());
+ disassembleInstr(instr2->InstructionBits());
+ disassembleInstr(instr3->InstructionBits());
+ disassembleInstr(instr4->InstructionBits());
+ disassembleInstr(instr5->InstructionBits());
+ disassembleInstr(instr6->InstructionBits());
+ disassembleInstr(instr7->InstructionBits());
+ MOZ_ASSERT(ExtractLoad64Value(pc) == value);
+ } else {
+ Instruction* instr0 = pc;
+ Instruction* instr2 = pc + 2 * kInstrSize;
+ Instruction* instr3 = pc + 3 * kInstrSize;
+ Instruction* instr4 = pc + 4 * kInstrSize;
+ Instruction* instr5 = pc + 5 * kInstrSize;
+ Instruction* instr6 = pc + 6 * kInstrSize;
+ Instruction* instr7 = pc + 7 * kInstrSize;
+ disassembleInstr(instr0->InstructionBits());
+ disassembleInstr(instr1->InstructionBits());
+ disassembleInstr(instr2->InstructionBits());
+ disassembleInstr(instr3->InstructionBits());
+ disassembleInstr(instr4->InstructionBits());
+ disassembleInstr(instr5->InstructionBits());
+ disassembleInstr(instr6->InstructionBits());
+ disassembleInstr(instr7->InstructionBits());
+ MOZ_ASSERT(IsAddi(*reinterpret_cast<Instr*>(instr1)));
+ set_target_value_at(pc, value);
+ }
+}
+
+void Assembler::set_target_value_at(Instruction* pc, uint64_t target) {
+ DEBUG_PRINTF("\tset_target_value_at: pc: %p\ttarget: %lx\n", pc, target);
+ uint32_t* p = reinterpret_cast<uint32_t*>(pc);
+ MOZ_ASSERT((target & 0xffff000000000000ll) == 0);
+#ifdef DEBUG
+ // Check we have the result from a li macro-instruction.
+ Instruction* instr0 = pc;
+ Instruction* instr1 = pc + 1 * kInstrSize;
+ Instruction* instr3 = pc + 3 * kInstrSize;
+ Instruction* instr5 = pc + 5 * kInstrSize;
+ MOZ_ASSERT(IsLui(*reinterpret_cast<Instr*>(instr0)) &&
+ IsAddi(*reinterpret_cast<Instr*>(instr1)) &&
+ IsOri(*reinterpret_cast<Instr*>(instr3)) &&
+ IsOri(*reinterpret_cast<Instr*>(instr5)));
+#endif
+ int64_t a6 = target & 0x3f; // bits 0:6. 6 bits
+ int64_t b11 = (target >> 6) & 0x7ff; // bits 6:11. 11 bits
+ int64_t high_31 = (target >> 17) & 0x7fffffff; // 31 bits
+ int64_t high_20 = ((high_31 + 0x800) >> 12); // 19 bits
+ int64_t low_12 = high_31 & 0xfff; // 12 bits
+ *p = *p & 0xfff;
+ *p = *p | ((int32_t)high_20 << 12);
+ *(p + 1) = *(p + 1) & 0xfffff;
+ *(p + 1) = *(p + 1) | ((int32_t)low_12 << 20);
+ *(p + 2) = *(p + 2) & 0xfffff;
+ *(p + 2) = *(p + 2) | (11 << 20);
+ *(p + 3) = *(p + 3) & 0xfffff;
+ *(p + 3) = *(p + 3) | ((int32_t)b11 << 20);
+ *(p + 4) = *(p + 4) & 0xfffff;
+ *(p + 4) = *(p + 4) | (6 << 20);
+ *(p + 5) = *(p + 5) & 0xfffff;
+ *(p + 5) = *(p + 5) | ((int32_t)a6 << 20);
+ MOZ_ASSERT(target_address_at(pc) == target);
+}
+
+void Assembler::WriteLoad64Instructions(Instruction* inst0, Register reg,
+ uint64_t value) {
+ DEBUG_PRINTF("\tWriteLoad64Instructions\n");
+ // Initialize rd with an address
+ // Pointers are 48 bits
+ // 6 fixed instructions are generated
+ MOZ_ASSERT((value & 0xfff0000000000000ll) == 0);
+ int64_t a6 = value & 0x3f; // bits 0:5. 6 bits
+ int64_t b11 = (value >> 6) & 0x7ff; // bits 6:11. 11 bits
+ int64_t high_31 = (value >> 17) & 0x7fffffff; // 31 bits
+ int64_t high_20 = ((high_31 + 0x800) >> 12); // 19 bits
+ int64_t low_12 = high_31 & 0xfff; // 12 bits
+ Instr lui_ = LUI | (reg.code() << kRdShift) |
+ ((int32_t)high_20 << kImm20Shift); // lui(rd, (int32_t)high_20);
+ *reinterpret_cast<Instr*>(inst0) = lui_;
+
+ Instr addi_ =
+ OP_IMM | (reg.code() << kRdShift) | (0b000 << kFunct3Shift) |
+ (reg.code() << kRs1Shift) |
+ (low_12 << kImm12Shift); // addi(rd, rd, low_12); // 31 bits in rd.
+ *reinterpret_cast<Instr*>(inst0 + 1 * kInstrSize) = addi_;
+
+ Instr slli_ =
+ OP_IMM | (reg.code() << kRdShift) | (0b001 << kFunct3Shift) |
+ (reg.code() << kRs1Shift) |
+ (11 << kImm12Shift); // slli(rd, rd, 11); // Space for next 11 bis
+ *reinterpret_cast<Instr*>(inst0 + 2 * kInstrSize) = slli_;
+
+ Instr ori_b11 = OP_IMM | (reg.code() << kRdShift) | (0b110 << kFunct3Shift) |
+ (reg.code() << kRs1Shift) |
+ (b11 << kImm12Shift); // ori(rd, rd, b11); // 11 bits
+ // are put in. 42 bit in rd
+ *reinterpret_cast<Instr*>(inst0 + 3 * kInstrSize) = ori_b11;
+
+ slli_ = OP_IMM | (reg.code() << kRdShift) | (0b001 << kFunct3Shift) |
+ (reg.code() << kRs1Shift) |
+ (6 << kImm12Shift); // slli(rd, rd, 6); // Space for next 11 bis
+ *reinterpret_cast<Instr*>(inst0 + 4 * kInstrSize) =
+ slli_; // slli(rd, rd, 6); // Space for next 6 bits
+
+ Instr ori_a6 = OP_IMM | (reg.code() << kRdShift) | (0b110 << kFunct3Shift) |
+ (reg.code() << kRs1Shift) |
+ (a6 << kImm12Shift); // ori(rd, rd, a6); // 6 bits are
+ // put in. 48 bis in rd
+ *reinterpret_cast<Instr*>(inst0 + 5 * kInstrSize) = ori_a6;
+ disassembleInstr((inst0 + 0 * kInstrSize)->InstructionBits());
+ disassembleInstr((inst0 + 1 * kInstrSize)->InstructionBits());
+ disassembleInstr((inst0 + 2 * kInstrSize)->InstructionBits());
+ disassembleInstr((inst0 + 3 * kInstrSize)->InstructionBits());
+ disassembleInstr((inst0 + 4 * kInstrSize)->InstructionBits());
+ disassembleInstr((inst0 + 5 * kInstrSize)->InstructionBits());
+ disassembleInstr((inst0 + 6 * kInstrSize)->InstructionBits());
+ MOZ_ASSERT(ExtractLoad64Value(inst0) == value);
+}
+
+// This just stomps over memory with 32 bits of raw data. Its purpose is to
+// overwrite the call of JITed code with 32 bits worth of an offset. This will
+// is only meant to function on code that has been invalidated, so it should
+// be totally safe. Since that instruction will never be executed again, a
+// ICache flush should not be necessary
+void Assembler::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) {
+ // Raw is going to be the return address.
+ uint32_t* raw = (uint32_t*)label.raw();
+ // Overwrite the 4 bytes before the return address, which will
+ // end up being the call instruction.
+ *(raw - 1) = imm.value;
+}
+
+void Assembler::target_at_put(BufferOffset pos, BufferOffset target_pos,
+ bool trampoline) {
+ if (m_buffer.oom()) {
+ return;
+ }
+ DEBUG_PRINTF("\ttarget_at_put: %p (%d) to %p (%d)\n",
+ reinterpret_cast<Instr*>(editSrc(pos)), pos.getOffset(),
+ reinterpret_cast<Instr*>(editSrc(pos)) + target_pos.getOffset() -
+ pos.getOffset(),
+ target_pos.getOffset());
+ Instruction* instruction = editSrc(pos);
+ Instr instr = instruction->InstructionBits();
+ switch (instruction->InstructionOpcodeType()) {
+ case BRANCH: {
+ instr = SetBranchOffset(pos.getOffset(), target_pos.getOffset(), instr);
+ instr_at_put(pos, instr);
+ } break;
+ case JAL: {
+ MOZ_ASSERT(IsJal(instr));
+ instr = SetJalOffset(pos.getOffset(), target_pos.getOffset(), instr);
+ instr_at_put(pos, instr);
+ } break;
+ case LUI: {
+ set_target_value_at(instruction,
+ reinterpret_cast<uintptr_t>(editSrc(target_pos)));
+ } break;
+ case AUIPC: {
+ Instr instr_auipc = instr;
+ Instr instr_I =
+ editSrc(BufferOffset(pos.getOffset() + 4))->InstructionBits();
+ MOZ_ASSERT(IsJalr(instr_I) || IsAddi(instr_I));
+
+ intptr_t offset = target_pos.getOffset() - pos.getOffset();
+ if (is_int21(offset) && IsJalr(instr_I) && trampoline) {
+ MOZ_ASSERT(is_int21(offset) && ((offset & 1) == 0));
+ Instr instr = JAL;
+ instr = SetJalOffset(pos.getOffset(), target_pos.getOffset(), instr);
+ MOZ_ASSERT(IsJal(instr));
+ MOZ_ASSERT(JumpOffset(instr) == offset);
+ instr_at_put(pos, instr);
+ instr_at_put(BufferOffset(pos.getOffset() + 4), kNopByte);
+ } else {
+ MOZ_RELEASE_ASSERT(is_int32(offset + 0x800));
+ MOZ_ASSERT(instruction->RdValue() ==
+ editSrc(BufferOffset(pos.getOffset() + 4))->Rs1Value());
+ int32_t Hi20 = (((int32_t)offset + 0x800) >> 12);
+ int32_t Lo12 = (int32_t)offset << 20 >> 20;
+
+ instr_auipc =
+ (instr_auipc & ~kImm31_12Mask) | ((Hi20 & kImm19_0Mask) << 12);
+ instr_at_put(pos, instr_auipc);
+
+ const int kImm31_20Mask = ((1 << 12) - 1) << 20;
+ const int kImm11_0Mask = ((1 << 12) - 1);
+ instr_I = (instr_I & ~kImm31_20Mask) | ((Lo12 & kImm11_0Mask) << 20);
+ instr_at_put(BufferOffset(pos.getOffset() + 4), instr_I);
+ }
+ } break;
+ default:
+ UNIMPLEMENTED_RISCV();
+ break;
+ }
+}
+
+const int kEndOfChain = -1;
+const int32_t kEndOfJumpChain = 0;
+
+int Assembler::target_at(BufferOffset pos, bool is_internal) {
+ if (oom()) {
+ return kEndOfChain;
+ }
+ Instruction* instruction = editSrc(pos);
+ Instruction* instruction2 = nullptr;
+ if (IsAuipc(instruction->InstructionBits())) {
+ instruction2 = editSrc(BufferOffset(pos.getOffset() + kInstrSize));
+ }
+ return target_at(instruction, pos, is_internal, instruction2);
+}
+
+int Assembler::target_at(Instruction* instruction, BufferOffset pos,
+ bool is_internal, Instruction* instruction2) {
+ DEBUG_PRINTF("\t target_at: %p(%x)\n\t",
+ reinterpret_cast<Instr*>(instruction), pos.getOffset());
+ disassembleInstr(instruction->InstructionBits());
+ Instr instr = instruction->InstructionBits();
+ switch (instruction->InstructionOpcodeType()) {
+ case BRANCH: {
+ int32_t imm13 = BranchOffset(instr);
+ if (imm13 == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ } else {
+ DEBUG_PRINTF("\t target_at: %d %d\n", imm13, pos.getOffset() + imm13);
+ return pos.getOffset() + imm13;
+ }
+ }
+ case JAL: {
+ int32_t imm21 = JumpOffset(instr);
+ if (imm21 == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ } else {
+ DEBUG_PRINTF("\t target_at: %d %d\n", imm21, pos.getOffset() + imm21);
+ return pos.getOffset() + imm21;
+ }
+ }
+ case JALR: {
+ int32_t imm12 = instr >> 20;
+ if (imm12 == kEndOfJumpChain) {
+ // EndOfChain sentinel is returned directly, not relative to pc or pos.
+ return kEndOfChain;
+ } else {
+ DEBUG_PRINTF("\t target_at: %d %d\n", imm12, pos.getOffset() + imm12);
+ return pos.getOffset() + imm12;
+ }
+ }
+ case LUI: {
+ uintptr_t imm = target_address_at(instruction);
+ uintptr_t instr_address = reinterpret_cast<uintptr_t>(instruction);
+ if (imm == kEndOfJumpChain) {
+ return kEndOfChain;
+ } else {
+ MOZ_ASSERT(instr_address - imm < INT_MAX);
+ int32_t delta = static_cast<int32_t>(instr_address - imm);
+ MOZ_ASSERT(pos.getOffset() > delta);
+ return pos.getOffset() - delta;
+ }
+ }
+ case AUIPC: {
+ MOZ_ASSERT(instruction2 != nullptr);
+ Instr instr_auipc = instr;
+ Instr instr_I = instruction2->InstructionBits();
+ MOZ_ASSERT(IsJalr(instr_I) || IsAddi(instr_I));
+ int32_t offset = BrachlongOffset(instr_auipc, instr_I);
+ if (offset == kEndOfJumpChain) return kEndOfChain;
+ DEBUG_PRINTF("\t target_at: %d %d\n", offset, pos.getOffset() + offset);
+ return offset + pos.getOffset();
+ }
+ default: {
+ UNIMPLEMENTED_RISCV();
+ }
+ }
+}
+
+uint32_t Assembler::next_link(Label* L, bool is_internal) {
+ MOZ_ASSERT(L->used());
+ BufferOffset pos(L);
+ int link = target_at(pos, is_internal);
+ if (link == kEndOfChain) {
+ L->reset();
+ return LabelBase::INVALID_OFFSET;
+ } else {
+ MOZ_ASSERT(link >= 0);
+ DEBUG_PRINTF("next: %p to offset %d\n", L, link);
+ L->use(link);
+ return link;
+ }
+}
+
+void Assembler::bind(Label* label, BufferOffset boff) {
+ JitSpew(JitSpew_Codegen, ".set Llabel %p %d", label, currentOffset());
+ DEBUG_PRINTF(".set Llabel %p\n", label);
+ // If our caller didn't give us an explicit target to bind to
+ // then we want to bind to the location of the next instruction
+ BufferOffset dest = boff.assigned() ? boff : nextOffset();
+ if (label->used()) {
+ uint32_t next;
+
+ // A used label holds a link to branch that uses it.
+ do {
+ BufferOffset b(label);
+ DEBUG_PRINTF("\tbind next:%d\n", b.getOffset());
+ // Even a 0 offset may be invalid if we're out of memory.
+ if (oom()) {
+ return;
+ }
+ int fixup_pos = b.getOffset();
+ int dist = dest.getOffset() - fixup_pos;
+ next = next_link(label, false);
+ DEBUG_PRINTF("\t%p fixup: %d next: %d\n", label, fixup_pos, next);
+ DEBUG_PRINTF("\t fixup: %d dest: %d dist: %d %d %d\n", fixup_pos,
+ dest.getOffset(), dist, nextOffset().getOffset(),
+ currentOffset());
+ Instruction* instruction = editSrc(b);
+ Instr instr = instruction->InstructionBits();
+ if (IsBranch(instr)) {
+ if (dist > kMaxBranchOffset) {
+ MOZ_ASSERT(next != LabelBase::INVALID_OFFSET);
+ MOZ_RELEASE_ASSERT((next - fixup_pos) <= kMaxBranchOffset);
+ MOZ_ASSERT(IsAuipc(editSrc(BufferOffset(next))->InstructionBits()));
+ MOZ_ASSERT(
+ IsJalr(editSrc(BufferOffset(next + 4))->InstructionBits()));
+ DEBUG_PRINTF("\t\ttrampolining: %d\n", next);
+ } else {
+ target_at_put(b, dest);
+ BufferOffset deadline(b.getOffset() +
+ ImmBranchMaxForwardOffset(CondBranchRangeType));
+ m_buffer.unregisterBranchDeadline(CondBranchRangeType, deadline);
+ }
+ } else if (IsJal(instr)) {
+ if (dist > kMaxJumpOffset) {
+ MOZ_ASSERT(next != LabelBase::INVALID_OFFSET);
+ MOZ_RELEASE_ASSERT((next - fixup_pos) <= kMaxJumpOffset);
+ MOZ_ASSERT(IsAuipc(editSrc(BufferOffset(next))->InstructionBits()));
+ MOZ_ASSERT(
+ IsJalr(editSrc(BufferOffset(next + 4))->InstructionBits()));
+ DEBUG_PRINTF("\t\ttrampolining: %d\n", next);
+ } else {
+ target_at_put(b, dest);
+ BufferOffset deadline(
+ b.getOffset() + ImmBranchMaxForwardOffset(UncondBranchRangeType));
+ m_buffer.unregisterBranchDeadline(UncondBranchRangeType, deadline);
+ }
+ } else {
+ MOZ_ASSERT(IsAuipc(instr));
+ target_at_put(b, dest);
+ }
+ } while (next != LabelBase::INVALID_OFFSET);
+ }
+ label->bind(dest.getOffset());
+}
+
+void Assembler::Bind(uint8_t* rawCode, const CodeLabel& label) {
+ if (label.patchAt().bound()) {
+ auto mode = label.linkMode();
+ intptr_t offset = label.patchAt().offset();
+ intptr_t target = label.target().offset();
+
+ if (mode == CodeLabel::RawPointer) {
+ *reinterpret_cast<const void**>(rawCode + offset) = rawCode + target;
+ } else {
+ MOZ_ASSERT(mode == CodeLabel::MoveImmediate ||
+ mode == CodeLabel::JumpImmediate);
+ Instruction* inst = (Instruction*)(rawCode + offset);
+ Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + target));
+ }
+ }
+}
+
+bool Assembler::is_near(Label* L) {
+ MOZ_ASSERT(L->bound());
+ return is_intn((currentOffset() - L->offset()), kJumpOffsetBits);
+}
+
+bool Assembler::is_near(Label* L, OffsetSize bits) {
+ if (L == nullptr || !L->bound()) return true;
+ return is_intn((currentOffset() - L->offset()), bits);
+}
+
+bool Assembler::is_near_branch(Label* L) {
+ MOZ_ASSERT(L->bound());
+ return is_intn((currentOffset() - L->offset()), kBranchOffsetBits);
+}
+
+int32_t Assembler::branch_long_offset(Label* L) {
+ if (oom()) {
+ return kEndOfJumpChain;
+ }
+ intptr_t target_pos;
+ BufferOffset next_instr_offset = nextInstrOffset(2);
+ DEBUG_PRINTF("\tbranch_long_offset: %p to (%d)\n", L,
+ next_instr_offset.getOffset());
+ if (L->bound()) {
+ JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
+ next_instr_offset.getOffset());
+ target_pos = L->offset();
+ } else {
+ if (L->used()) {
+ LabelCahe::Ptr p = label_cache_.lookup(L->offset());
+ MOZ_ASSERT(p);
+ MOZ_ASSERT(p->key() == L->offset());
+ target_pos = p->value().getOffset();
+ target_at_put(BufferOffset(target_pos), next_instr_offset);
+ DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
+ next_instr_offset.getOffset());
+ bool ok = label_cache_.put(L->offset(), next_instr_offset);
+ if (!ok) {
+ NoEnoughLabelCache();
+ }
+ return kEndOfJumpChain;
+ } else {
+ JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
+ next_instr_offset.getOffset());
+ L->use(next_instr_offset.getOffset());
+ DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
+ next_instr_offset.getOffset());
+ bool ok = label_cache_.putNew(L->offset(), next_instr_offset);
+ if (!ok) {
+ NoEnoughLabelCache();
+ }
+ return kEndOfJumpChain;
+ }
+ }
+ intptr_t offset = target_pos - next_instr_offset.getOffset();
+ MOZ_ASSERT((offset & 3) == 0);
+ MOZ_ASSERT(is_int32(offset));
+ return static_cast<int32_t>(offset);
+}
+
+int32_t Assembler::branch_offset_helper(Label* L, OffsetSize bits) {
+ if (oom()) {
+ return kEndOfJumpChain;
+ }
+ int32_t target_pos;
+ BufferOffset next_instr_offset = nextInstrOffset();
+ DEBUG_PRINTF("\tbranch_offset_helper: %p to %d\n", L,
+ next_instr_offset.getOffset());
+ // This is the last possible branch target.
+ if (L->bound()) {
+ JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
+ next_instr_offset.getOffset());
+ target_pos = L->offset();
+ } else {
+ BufferOffset deadline(next_instr_offset.getOffset() +
+ ImmBranchMaxForwardOffset(bits));
+ DEBUG_PRINTF("\tregisterBranchDeadline %d type %d\n", deadline.getOffset(),
+ OffsetSizeToImmBranchRangeType(bits));
+ m_buffer.registerBranchDeadline(OffsetSizeToImmBranchRangeType(bits),
+ deadline);
+ if (L->used()) {
+ LabelCahe::Ptr p = label_cache_.lookup(L->offset());
+ MOZ_ASSERT(p);
+ MOZ_ASSERT(p->key() == L->offset());
+ target_pos = p->value().getOffset();
+ target_at_put(BufferOffset(target_pos), next_instr_offset);
+ DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
+ next_instr_offset.getOffset());
+ bool ok = label_cache_.put(L->offset(), next_instr_offset);
+ if (!ok) {
+ NoEnoughLabelCache();
+ }
+ return kEndOfJumpChain;
+ } else {
+ JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L,
+ next_instr_offset.getOffset());
+ L->use(next_instr_offset.getOffset());
+ bool ok = label_cache_.putNew(L->offset(), next_instr_offset);
+ if (!ok) {
+ NoEnoughLabelCache();
+ }
+ DEBUG_PRINTF("\tLabel %p added to link: %d\n", L,
+ next_instr_offset.getOffset());
+ return kEndOfJumpChain;
+ }
+ }
+
+ int32_t offset = target_pos - next_instr_offset.getOffset();
+ DEBUG_PRINTF("\toffset = %d\n", offset);
+ MOZ_ASSERT(is_intn(offset, bits));
+ MOZ_ASSERT((offset & 1) == 0);
+ return offset;
+}
+
+Assembler::Condition Assembler::InvertCondition(Condition cond) {
+ switch (cond) {
+ case Equal:
+ return NotEqual;
+ case NotEqual:
+ return Equal;
+ case Zero:
+ return NonZero;
+ case NonZero:
+ return Zero;
+ case LessThan:
+ return GreaterThanOrEqual;
+ case LessThanOrEqual:
+ return GreaterThan;
+ case GreaterThan:
+ return LessThanOrEqual;
+ case GreaterThanOrEqual:
+ return LessThan;
+ case Above:
+ return BelowOrEqual;
+ case AboveOrEqual:
+ return Below;
+ case Below:
+ return AboveOrEqual;
+ case BelowOrEqual:
+ return Above;
+ case Signed:
+ return NotSigned;
+ case NotSigned:
+ return Signed;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+Assembler::DoubleCondition Assembler::InvertCondition(DoubleCondition cond) {
+ switch (cond) {
+ case DoubleOrdered:
+ return DoubleUnordered;
+ case DoubleEqual:
+ return DoubleNotEqualOrUnordered;
+ case DoubleNotEqual:
+ return DoubleEqualOrUnordered;
+ case DoubleGreaterThan:
+ return DoubleLessThanOrEqualOrUnordered;
+ case DoubleGreaterThanOrEqual:
+ return DoubleLessThanOrUnordered;
+ case DoubleLessThan:
+ return DoubleGreaterThanOrEqualOrUnordered;
+ case DoubleLessThanOrEqual:
+ return DoubleGreaterThanOrUnordered;
+ case DoubleUnordered:
+ return DoubleOrdered;
+ case DoubleEqualOrUnordered:
+ return DoubleNotEqual;
+ case DoubleNotEqualOrUnordered:
+ return DoubleEqual;
+ case DoubleGreaterThanOrUnordered:
+ return DoubleLessThanOrEqual;
+ case DoubleGreaterThanOrEqualOrUnordered:
+ return DoubleLessThan;
+ case DoubleLessThanOrUnordered:
+ return DoubleGreaterThanOrEqual;
+ case DoubleLessThanOrEqualOrUnordered:
+ return DoubleGreaterThan;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+// Break / Trap instructions.
+void Assembler::break_(uint32_t code, bool break_as_stop) {
+ // We need to invalidate breaks that could be stops as well because the
+ // simulator expects a char pointer after the stop instruction.
+ // See constants-mips.h for explanation.
+ MOZ_ASSERT(
+ (break_as_stop && code <= kMaxStopCode && code > kMaxTracepointCode) ||
+ (!break_as_stop && (code > kMaxStopCode || code <= kMaxTracepointCode)));
+
+ // since ebreak does not allow additional immediate field, we use the
+ // immediate field of lui instruction immediately following the ebreak to
+ // encode the "code" info
+ ebreak();
+ MOZ_ASSERT(is_uint20(code));
+ lui(zero_reg, code);
+}
+
+void Assembler::ToggleToJmp(CodeLocationLabel inst_) {
+ Instruction* inst = (Instruction*)inst_.raw();
+ MOZ_ASSERT(IsAddi(inst->InstructionBits()));
+ int32_t offset = inst->Imm12Value();
+ MOZ_ASSERT(is_int12(offset));
+ Instr jal_ = JAL | (0b000 << kFunct3Shift) |
+ (offset & 0xff000) | // bits 19-12
+ ((offset & 0x800) << 9) | // bit 11
+ ((offset & 0x7fe) << 20) | // bits 10-1
+ ((offset & 0x100000) << 11); // bit 20
+ // jal(zero, offset);
+ *reinterpret_cast<Instr*>(inst) = jal_;
+}
+
+void Assembler::ToggleToCmp(CodeLocationLabel inst_) {
+ Instruction* inst = (Instruction*)inst_.raw();
+
+ // toggledJump is allways used for short jumps.
+ MOZ_ASSERT(IsJal(inst->InstructionBits()));
+ // Replace "jal zero_reg, offset" with "addi $zero, $zero, offset"
+ int32_t offset = inst->Imm20JValue();
+ MOZ_ASSERT(is_int12(offset));
+ Instr addi_ = OP_IMM | (0b000 << kFunct3Shift) |
+ (offset << kImm12Shift); // addi(zero, zero, low_12);
+ *reinterpret_cast<Instr*>(inst) = addi_;
+}
+
+bool Assembler::reserve(size_t size) {
+ // This buffer uses fixed-size chunks so there's no point in reserving
+ // now vs. on-demand.
+ return !oom();
+}
+
+static JitCode* CodeFromJump(Instruction* jump) {
+ uint8_t* target = (uint8_t*)Assembler::ExtractLoad64Value(jump);
+ return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ while (reader.more()) {
+ JitCode* child =
+ CodeFromJump((Instruction*)(code->raw() + reader.readUnsigned()));
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ }
+}
+
+static void TraceOneDataRelocation(JSTracer* trc,
+ mozilla::Maybe<AutoWritableJitCode>& awjc,
+ JitCode* code, Instruction* inst) {
+ void* ptr = (void*)Assembler::ExtractLoad64Value(inst);
+ void* prior = ptr;
+
+ // Data relocations can be for Values or for raw pointers. If a Value is
+ // zero-tagged, we can trace it as if it were a raw pointer. If a Value
+ // is not zero-tagged, we have to interpret it as a Value to ensure that the
+ // tag bits are masked off to recover the actual pointer.
+ uintptr_t word = reinterpret_cast<uintptr_t>(ptr);
+ if (word >> JSVAL_TAG_SHIFT) {
+ // This relocation is a Value with a non-zero tag.
+ Value v = Value::fromRawBits(word);
+ TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value");
+ ptr = (void*)v.bitsAsPunboxPointer();
+ } else {
+ // This relocation is a raw pointer or a Value with a zero tag.
+ // No barrier needed since these are constants.
+ TraceManuallyBarrieredGenericPointerEdge(
+ trc, reinterpret_cast<gc::Cell**>(&ptr), "jit-masm-ptr");
+ }
+
+ if (ptr != prior) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ Assembler::UpdateLoad64Value(inst, uint64_t(ptr));
+ }
+}
+
+/* static */
+void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ mozilla::Maybe<AutoWritableJitCode> awjc;
+ while (reader.more()) {
+ size_t offset = reader.readUnsigned();
+ Instruction* inst = (Instruction*)(code->raw() + offset);
+ TraceOneDataRelocation(trc, awjc, code, inst);
+ }
+}
+
+UseScratchRegisterScope::UseScratchRegisterScope(Assembler* assembler)
+ : available_(assembler->GetScratchRegisterList()),
+ old_available_(*available_) {}
+
+UseScratchRegisterScope::~UseScratchRegisterScope() {
+ *available_ = old_available_;
+}
+
+Register UseScratchRegisterScope::Acquire() {
+ MOZ_ASSERT(available_ != nullptr);
+ MOZ_ASSERT(!available_->empty());
+ Register index = GeneralRegisterSet::FirstRegister(available_->bits());
+ available_->takeRegisterIndex(index);
+ return index;
+}
+
+bool UseScratchRegisterScope::hasAvailable() const {
+ return (available_->size()) != 0;
+}
+
+void Assembler::retarget(Label* label, Label* target) {
+ spew("retarget %p -> %p", label, target);
+ if (label->used() && !oom()) {
+ if (target->bound()) {
+ bind(label, BufferOffset(target));
+ } else if (target->used()) {
+ // The target is not bound but used. Prepend label's branch list
+ // onto target's.
+ int32_t next;
+ BufferOffset labelBranchOffset(label);
+
+ // Find the head of the use chain for label.
+ do {
+ next = next_link(label, false);
+ labelBranchOffset = BufferOffset(next);
+ } while (next != LabelBase::INVALID_OFFSET);
+
+ // Then patch the head of label's use chain to the tail of
+ // target's use chain, prepending the entire use chain of target.
+ target->use(label->offset());
+ target_at_put(labelBranchOffset, BufferOffset(target));
+ MOZ_CRASH("check");
+ } else {
+ // The target is unbound and unused. We can just take the head of
+ // the list hanging off of label, and dump that into target.
+ target->use(label->offset());
+ }
+ }
+ label->reset();
+}
+
+bool Assembler::appendRawCode(const uint8_t* code, size_t numBytes) {
+ if (m_buffer.oom()) {
+ return false;
+ }
+ while (numBytes > SliceSize) {
+ m_buffer.putBytes(SliceSize, code);
+ numBytes -= SliceSize;
+ code += SliceSize;
+ }
+ m_buffer.putBytes(numBytes, code);
+ return !m_buffer.oom();
+}
+
+void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) {
+ Instruction* i0 = (Instruction*)inst_.raw();
+ Instruction* i1 = (Instruction*)(inst_.raw() + 1 * kInstrSize);
+ Instruction* i2 = (Instruction*)(inst_.raw() + 2 * kInstrSize);
+ Instruction* i3 = (Instruction*)(inst_.raw() + 3 * kInstrSize);
+ Instruction* i4 = (Instruction*)(inst_.raw() + 4 * kInstrSize);
+ Instruction* i5 = (Instruction*)(inst_.raw() + 5 * kInstrSize);
+ Instruction* i6 = (Instruction*)(inst_.raw() + 6 * kInstrSize);
+
+ MOZ_ASSERT(IsLui(i0->InstructionBits()));
+ MOZ_ASSERT(IsAddi(i1->InstructionBits()));
+ MOZ_ASSERT(IsSlli(i2->InstructionBits()));
+ MOZ_ASSERT(IsOri(i3->InstructionBits()));
+ MOZ_ASSERT(IsSlli(i4->InstructionBits()));
+ MOZ_ASSERT(IsOri(i5->InstructionBits()));
+ if (enabled) {
+ Instr jalr_ = JALR | (ra.code() << kRdShift) | (0x0 << kFunct3Shift) |
+ (i5->RdValue() << kRs1Shift) | (0x0 << kImm12Shift);
+ *((Instr*)i6) = jalr_;
+ } else {
+ *((Instr*)i6) = kNopByte;
+ }
+}
+
+void Assembler::PatchShortRangeBranchToVeneer(Buffer* buffer, unsigned rangeIdx,
+ BufferOffset deadline,
+ BufferOffset veneer) {
+ if (buffer->oom()) {
+ return;
+ }
+ DEBUG_PRINTF("\tPatchShortRangeBranchToVeneer\n");
+ // Reconstruct the position of the branch from (rangeIdx, deadline).
+ ImmBranchRangeType branchRange = static_cast<ImmBranchRangeType>(rangeIdx);
+ BufferOffset branch(deadline.getOffset() -
+ ImmBranchMaxForwardOffset(branchRange));
+ Instruction* branchInst = buffer->getInst(branch);
+ Instruction* veneerInst_1 = buffer->getInst(veneer);
+ Instruction* veneerInst_2 =
+ buffer->getInst(BufferOffset(veneer.getOffset() + 4));
+ // Verify that the branch range matches what's encoded.
+ DEBUG_PRINTF("\t%p(%x): ", branchInst, branch.getOffset());
+ disassembleInstr(branchInst->InstructionBits(), JitSpew_Codegen);
+ DEBUG_PRINTF("\t instert veneer %x, branch:%x deadline: %x\n",
+ veneer.getOffset(), branch.getOffset(), deadline.getOffset());
+ MOZ_ASSERT(branchRange <= UncondBranchRangeType);
+ MOZ_ASSERT(branchInst->GetImmBranchRangeType() == branchRange);
+ // emit a long jump slot
+ Instr auipc = AUIPC | (t6.code() << kRdShift) | (0x0 << kImm20Shift);
+ Instr jalr = JALR | (zero_reg.code() << kRdShift) | (0x0 << kFunct3Shift) |
+ (t6.code() << kRs1Shift) | (0x0 << kImm12Shift);
+
+ // We want to insert veneer after branch in the linked list of instructions
+ // that use the same unbound label.
+ // The veneer should be an unconditional branch.
+ int32_t nextElemOffset = target_at(buffer->getInst(branch), branch, false);
+ int32_t dist;
+ // If offset is 0, this is the end of the linked list.
+ if (nextElemOffset != kEndOfChain) {
+ // Make the offset relative to veneer so it targets the same instruction
+ // as branchInst.
+ dist = nextElemOffset - veneer.getOffset();
+ } else {
+ dist = 0;
+ }
+ int32_t Hi20 = (((int32_t)dist + 0x800) >> 12);
+ int32_t Lo12 = (int32_t)dist << 20 >> 20;
+ auipc = SetAuipcOffset(Hi20, auipc);
+ jalr = SetJalrOffset(Lo12, jalr);
+ // insert veneer
+ veneerInst_1->SetInstructionBits(auipc);
+ veneerInst_2->SetInstructionBits(jalr);
+ // Now link branchInst to veneer.
+ if (IsBranch(branchInst->InstructionBits())) {
+ branchInst->SetInstructionBits(SetBranchOffset(
+ branch.getOffset(), veneer.getOffset(), branchInst->InstructionBits()));
+ } else {
+ MOZ_ASSERT(IsJal(branchInst->InstructionBits()));
+ branchInst->SetInstructionBits(SetJalOffset(
+ branch.getOffset(), veneer.getOffset(), branchInst->InstructionBits()));
+ }
+ DEBUG_PRINTF("\tfix to veneer:");
+ disassembleInstr(branchInst->InstructionBits());
+}
+} // namespace jit
+} // namespace js