diff options
Diffstat (limited to 'js/src/jit/riscv64/Assembler-riscv64.cpp')
| -rw-r--r-- | js/src/jit/riscv64/Assembler-riscv64.cpp | 1548 |
1 files changed, 1548 insertions, 0 deletions
diff --git a/js/src/jit/riscv64/Assembler-riscv64.cpp b/js/src/jit/riscv64/Assembler-riscv64.cpp new file mode 100644 index 0000000000..5370192987 --- /dev/null +++ b/js/src/jit/riscv64/Assembler-riscv64.cpp @@ -0,0 +1,1548 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Copyright (c) 1994-2006 Sun Microsystems Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// - Redistribution in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// - Neither the name of Sun Microsystems or the names of contributors may +// be used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// The original source code covered by the above license above has been +// modified significantly by Google Inc. +// Copyright 2021 the V8 project authors. All rights reserved. +#include "jit/riscv64/Assembler-riscv64.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/Maybe.h" + +#include "gc/Marking.h" +#include "jit/AutoWritableJitCode.h" +#include "jit/ExecutableAllocator.h" +#include "jit/riscv64/disasm/Disasm-riscv64.h" +#include "vm/Realm.h" + +using mozilla::DebugOnly; +namespace js { +namespace jit { + +#define UNIMPLEMENTED_RISCV() MOZ_CRASH("RISC_V not implemented"); + +bool Assembler::FLAG_riscv_debug = false; + +void Assembler::nop() { addi(ToRegister(0), ToRegister(0), 0); } + +// Size of the instruction stream, in bytes. +size_t Assembler::size() const { return m_buffer.size(); } + +bool Assembler::swapBuffer(wasm::Bytes& bytes) { + // For now, specialize to the one use case. As long as wasm::Bytes is a + // Vector, not a linked-list of chunks, there's not much we can do other + // than copy. + MOZ_ASSERT(bytes.empty()); + if (!bytes.resize(bytesNeeded())) { + return false; + } + m_buffer.executableCopy(bytes.begin()); + return true; +} + +// Size of the relocation table, in bytes. +size_t Assembler::jumpRelocationTableBytes() const { + return jumpRelocations_.length(); +} + +size_t Assembler::dataRelocationTableBytes() const { + return dataRelocations_.length(); +} +// Size of the data table, in bytes. +size_t Assembler::bytesNeeded() const { + return size() + jumpRelocationTableBytes() + dataRelocationTableBytes(); +} + +void Assembler::executableCopy(uint8_t* buffer) { + MOZ_ASSERT(isFinished); + m_buffer.executableCopy(buffer); +} + +uint32_t Assembler::AsmPoolMaxOffset = 1024; + +uint32_t Assembler::GetPoolMaxOffset() { + static bool isSet = false; + if (!isSet) { + char* poolMaxOffsetStr = getenv("ASM_POOL_MAX_OFFSET"); + uint32_t poolMaxOffset; + if (poolMaxOffsetStr && + sscanf(poolMaxOffsetStr, "%u", &poolMaxOffset) == 1) { + AsmPoolMaxOffset = poolMaxOffset; + } + isSet = true; + } + return AsmPoolMaxOffset; +} + +// Pool callbacks stuff: +void Assembler::InsertIndexIntoTag(uint8_t* load_, uint32_t index) { + MOZ_CRASH("Unimplement"); +} + +void Assembler::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr) { + MOZ_CRASH("Unimplement"); +} + +void Assembler::processCodeLabels(uint8_t* rawCode) { + for (const CodeLabel& label : codeLabels_) { + Bind(rawCode, label); + } +} + +void Assembler::WritePoolGuard(BufferOffset branch, Instruction* dest, + BufferOffset afterPool) { + DEBUG_PRINTF("\tWritePoolGuard\n"); + int32_t off = afterPool.getOffset() - branch.getOffset(); + if (!is_int21(off) || !((off & 0x1) == 0)) { + printf("%d\n", off); + MOZ_CRASH("imm invalid"); + } + // JAL encode is + // 31 | 30 21 | 20 | 19 12 | 11 7 | 6 0 | + // imm[20] | imm[10:1] | imm[11] | imm[19:12] | rd | opcode| + // 1 10 1 8 5 7 + // offset[20:1] dest JAL + int32_t imm20 = (off & 0xff000) | // bits 19-12 + ((off & 0x800) << 9) | // bit 11 + ((off & 0x7fe) << 20) | // bits 10-1 + ((off & 0x100000) << 11); // bit 20 + Instr instr = JAL | (imm20 & kImm20Mask); + dest->SetInstructionBits(instr); + DEBUG_PRINTF("%p(%x): ", dest, branch.getOffset()); + disassembleInstr(dest->InstructionBits(), JitSpew_Codegen); +} + +void Assembler::WritePoolHeader(uint8_t* start, Pool* p, bool isNatural) { + static_assert(sizeof(PoolHeader) == 4); + + // Get the total size of the pool. + const uintptr_t totalPoolSize = sizeof(PoolHeader) + p->getPoolSize(); + const uintptr_t totalPoolInstructions = totalPoolSize / kInstrSize; + + MOZ_ASSERT((totalPoolSize & 0x3) == 0); + MOZ_ASSERT(totalPoolInstructions < (1 << 15)); + + PoolHeader header(totalPoolInstructions, isNatural); + *(PoolHeader*)start = header; +} + +void Assembler::copyJumpRelocationTable(uint8_t* dest) { + if (jumpRelocations_.length()) { + memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length()); + } +} + +void Assembler::copyDataRelocationTable(uint8_t* dest) { + if (dataRelocations_.length()) { + memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length()); + } +} + +void Assembler::RV_li(Register rd, int64_t imm) { + UseScratchRegisterScope temps(this); + if (RecursiveLiCount(imm) > GeneralLiCount(imm, temps.hasAvailable())) { + GeneralLi(rd, imm); + } else { + RecursiveLi(rd, imm); + } +} + +int Assembler::RV_li_count(int64_t imm, bool is_get_temp_reg) { + if (RecursiveLiCount(imm) > GeneralLiCount(imm, is_get_temp_reg)) { + return GeneralLiCount(imm, is_get_temp_reg); + } else { + return RecursiveLiCount(imm); + } +} + +void Assembler::GeneralLi(Register rd, int64_t imm) { + // 64-bit imm is put in the register rd. + // In most cases the imm is 32 bit and 2 instructions are generated. If a + // temporary register is available, in the worst case, 6 instructions are + // generated for a full 64-bit immediate. If temporay register is not + // available the maximum will be 8 instructions. If imm is more than 32 bits + // and a temp register is available, imm is divided into two 32-bit parts, + // low_32 and up_32. Each part is built in a separate register. low_32 is + // built before up_32. If low_32 is negative (upper 32 bits are 1), 0xffffffff + // is subtracted from up_32 before up_32 is built. This compensates for 32 + // bits of 1's in the lower when the two registers are added. If no temp is + // available, the upper 32 bit is built in rd, and the lower 32 bits are + // devided to 3 parts (11, 11, and 10 bits). The parts are shifted and added + // to the upper part built in rd. + if (is_int32(imm + 0x800)) { + // 32-bit case. Maximum of 2 instructions generated + int64_t high_20 = ((imm + 0x800) >> 12); + int64_t low_12 = imm << 52 >> 52; + if (high_20) { + lui(rd, (int32_t)high_20); + if (low_12) { + addi(rd, rd, low_12); + } + } else { + addi(rd, zero_reg, low_12); + } + return; + } else { + UseScratchRegisterScope temps(this); + BlockTrampolinePoolScope block_trampoline_pool(this, 8); + // 64-bit case: divide imm into two 32-bit parts, upper and lower + int64_t up_32 = imm >> 32; + int64_t low_32 = imm & 0xffffffffull; + Register temp_reg = rd; + // Check if a temporary register is available + if (up_32 == 0 || low_32 == 0) { + // No temp register is needed + } else { + temp_reg = temps.hasAvailable() ? temps.Acquire() : InvalidReg; + } + if (temp_reg != InvalidReg) { + // keep track of hardware behavior for lower part in sim_low + int64_t sim_low = 0; + // Build lower part + if (low_32 != 0) { + int64_t high_20 = ((low_32 + 0x800) >> 12); + int64_t low_12 = low_32 & 0xfff; + if (high_20) { + // Adjust to 20 bits for the case of overflow + high_20 &= 0xfffff; + sim_low = ((high_20 << 12) << 32) >> 32; + lui(rd, (int32_t)high_20); + if (low_12) { + sim_low += (low_12 << 52 >> 52) | low_12; + addi(rd, rd, low_12); + } + } else { + sim_low = low_12; + ori(rd, zero_reg, low_12); + } + } + if (sim_low & 0x100000000) { + // Bit 31 is 1. Either an overflow or a negative 64 bit + if (up_32 == 0) { + // Positive number, but overflow because of the add 0x800 + slli(rd, rd, 32); + srli(rd, rd, 32); + return; + } + // low_32 is a negative 64 bit after the build + up_32 = (up_32 - 0xffffffff) & 0xffffffff; + } + if (up_32 == 0) { + return; + } + // Build upper part in a temporary register + if (low_32 == 0) { + // Build upper part in rd + temp_reg = rd; + } + int64_t high_20 = (up_32 + 0x800) >> 12; + int64_t low_12 = up_32 & 0xfff; + if (high_20) { + // Adjust to 20 bits for the case of overflow + high_20 &= 0xfffff; + lui(temp_reg, (int32_t)high_20); + if (low_12) { + addi(temp_reg, temp_reg, low_12); + } + } else { + ori(temp_reg, zero_reg, low_12); + } + // Put it at the bgining of register + slli(temp_reg, temp_reg, 32); + if (low_32 != 0) { + add(rd, rd, temp_reg); + } + return; + } + // No temp register. Build imm in rd. + // Build upper 32 bits first in rd. Divide lower 32 bits parts and add + // parts to the upper part by doing shift and add. + // First build upper part in rd. + int64_t high_20 = (up_32 + 0x800) >> 12; + int64_t low_12 = up_32 & 0xfff; + if (high_20) { + // Adjust to 20 bits for the case of overflow + high_20 &= 0xfffff; + lui(rd, (int32_t)high_20); + if (low_12) { + addi(rd, rd, low_12); + } + } else { + ori(rd, zero_reg, low_12); + } + // upper part already in rd. Each part to be added to rd, has maximum of 11 + // bits, and always starts with a 1. rd is shifted by the size of the part + // plus the number of zeros between the parts. Each part is added after the + // left shift. + uint32_t mask = 0x80000000; + int32_t shift_val = 0; + int32_t i; + for (i = 0; i < 32; i++) { + if ((low_32 & mask) == 0) { + mask >>= 1; + shift_val++; + if (i == 31) { + // rest is zero + slli(rd, rd, shift_val); + } + continue; + } + // The first 1 seen + int32_t part; + if ((i + 11) < 32) { + // Pick 11 bits + part = ((uint32_t)(low_32 << i) >> i) >> (32 - (i + 11)); + slli(rd, rd, shift_val + 11); + ori(rd, rd, part); + i += 10; + mask >>= 11; + } else { + part = (uint32_t)(low_32 << i) >> i; + slli(rd, rd, shift_val + (32 - i)); + ori(rd, rd, part); + break; + } + shift_val = 0; + } + } +} + +int Assembler::GeneralLiCount(int64_t imm, bool is_get_temp_reg) { + int count = 0; + // imitate Assembler::RV_li + if (is_int32(imm + 0x800)) { + // 32-bit case. Maximum of 2 instructions generated + int64_t high_20 = ((imm + 0x800) >> 12); + int64_t low_12 = imm << 52 >> 52; + if (high_20) { + count++; + if (low_12) { + count++; + } + } else { + count++; + } + return count; + } else { + // 64-bit case: divide imm into two 32-bit parts, upper and lower + int64_t up_32 = imm >> 32; + int64_t low_32 = imm & 0xffffffffull; + // Check if a temporary register is available + if (is_get_temp_reg) { + // keep track of hardware behavior for lower part in sim_low + int64_t sim_low = 0; + // Build lower part + if (low_32 != 0) { + int64_t high_20 = ((low_32 + 0x800) >> 12); + int64_t low_12 = low_32 & 0xfff; + if (high_20) { + // Adjust to 20 bits for the case of overflow + high_20 &= 0xfffff; + sim_low = ((high_20 << 12) << 32) >> 32; + count++; + if (low_12) { + sim_low += (low_12 << 52 >> 52) | low_12; + count++; + } + } else { + sim_low = low_12; + count++; + } + } + if (sim_low & 0x100000000) { + // Bit 31 is 1. Either an overflow or a negative 64 bit + if (up_32 == 0) { + // Positive number, but overflow because of the add 0x800 + count++; + count++; + return count; + } + // low_32 is a negative 64 bit after the build + up_32 = (up_32 - 0xffffffff) & 0xffffffff; + } + if (up_32 == 0) { + return count; + } + int64_t high_20 = (up_32 + 0x800) >> 12; + int64_t low_12 = up_32 & 0xfff; + if (high_20) { + // Adjust to 20 bits for the case of overflow + high_20 &= 0xfffff; + count++; + if (low_12) { + count++; + } + } else { + count++; + } + // Put it at the bgining of register + count++; + if (low_32 != 0) { + count++; + } + return count; + } + // No temp register. Build imm in rd. + // Build upper 32 bits first in rd. Divide lower 32 bits parts and add + // parts to the upper part by doing shift and add. + // First build upper part in rd. + int64_t high_20 = (up_32 + 0x800) >> 12; + int64_t low_12 = up_32 & 0xfff; + if (high_20) { + // Adjust to 20 bits for the case of overflow + high_20 &= 0xfffff; + count++; + if (low_12) { + count++; + } + } else { + count++; + } + // upper part already in rd. Each part to be added to rd, has maximum of 11 + // bits, and always starts with a 1. rd is shifted by the size of the part + // plus the number of zeros between the parts. Each part is added after the + // left shift. + uint32_t mask = 0x80000000; + int32_t i; + for (i = 0; i < 32; i++) { + if ((low_32 & mask) == 0) { + mask >>= 1; + if (i == 31) { + // rest is zero + count++; + } + continue; + } + // The first 1 seen + if ((i + 11) < 32) { + // Pick 11 bits + count++; + count++; + i += 10; + mask >>= 11; + } else { + count++; + count++; + break; + } + } + } + return count; +} + +void Assembler::li_ptr(Register rd, int64_t imm) { + m_buffer.enterNoNops(); + m_buffer.assertNoPoolAndNoNops(); + // Initialize rd with an address + // Pointers are 48 bits + // 6 fixed instructions are generated + DEBUG_PRINTF("li_ptr(%d, %lx <%ld>)\n", ToNumber(rd), imm, imm); + MOZ_ASSERT((imm & 0xfff0000000000000ll) == 0); + int64_t a6 = imm & 0x3f; // bits 0:5. 6 bits + int64_t b11 = (imm >> 6) & 0x7ff; // bits 6:11. 11 bits + int64_t high_31 = (imm >> 17) & 0x7fffffff; // 31 bits + int64_t high_20 = ((high_31 + 0x800) >> 12); // 19 bits + int64_t low_12 = high_31 & 0xfff; // 12 bits + lui(rd, (int32_t)high_20); + addi(rd, rd, low_12); // 31 bits in rd. + slli(rd, rd, 11); // Space for next 11 bis + ori(rd, rd, b11); // 11 bits are put in. 42 bit in rd + slli(rd, rd, 6); // Space for next 6 bits + ori(rd, rd, a6); // 6 bits are put in. 48 bis in rd + m_buffer.leaveNoNops(); +} + +void Assembler::li_constant(Register rd, int64_t imm) { + m_buffer.enterNoNops(); + m_buffer.assertNoPoolAndNoNops(); + DEBUG_PRINTF("li_constant(%d, %lx <%ld>)\n", ToNumber(rd), imm, imm); + lui(rd, (imm + (1LL << 47) + (1LL << 35) + (1LL << 23) + (1LL << 11)) >> + 48); // Bits 63:48 + addiw(rd, rd, + (imm + (1LL << 35) + (1LL << 23) + (1LL << 11)) << 16 >> + 52); // Bits 47:36 + slli(rd, rd, 12); + addi(rd, rd, (imm + (1LL << 23) + (1LL << 11)) << 28 >> 52); // Bits 35:24 + slli(rd, rd, 12); + addi(rd, rd, (imm + (1LL << 11)) << 40 >> 52); // Bits 23:12 + slli(rd, rd, 12); + addi(rd, rd, imm << 52 >> 52); // Bits 11:0 + m_buffer.leaveNoNops(); +} + +ABIArg ABIArgGenerator::next(MIRType type) { + switch (type) { + case MIRType::Int32: + case MIRType::Int64: + case MIRType::Pointer: + case MIRType::WasmAnyRef: + case MIRType::StackResults: { + if (intRegIndex_ == NumIntArgRegs) { + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(uintptr_t); + break; + } + current_ = ABIArg(Register::FromCode(intRegIndex_ + a0.encoding())); + intRegIndex_++; + break; + } + case MIRType::Float32: + case MIRType::Double: { + if (floatRegIndex_ == NumFloatArgRegs) { + current_ = ABIArg(stackOffset_); + stackOffset_ += sizeof(double); + break; + } + current_ = ABIArg(FloatRegister( + FloatRegisters::Encoding(floatRegIndex_ + fa0.encoding()), + type == MIRType::Double ? FloatRegisters::Double + : FloatRegisters::Single)); + floatRegIndex_++; + break; + } + case MIRType::Simd128: { + MOZ_CRASH("RISCV64 does not support simd yet."); + break; + } + default: + MOZ_CRASH("Unexpected argument type"); + } + return current_; +} + +bool Assembler::oom() const { + return AssemblerShared::oom() || m_buffer.oom() || jumpRelocations_.oom() || + dataRelocations_.oom() || !enoughLabelCache_; +} + +int Assembler::disassembleInstr(Instr instr, bool enable_spew) { + if (!FLAG_riscv_debug && !enable_spew) return -1; + disasm::NameConverter converter; + disasm::Disassembler disasm(converter); + EmbeddedVector<char, 128> disasm_buffer; + + int size = + disasm.InstructionDecode(disasm_buffer, reinterpret_cast<byte*>(&instr)); + DEBUG_PRINTF("%s\n", disasm_buffer.start()); + if (enable_spew) { + JitSpew(JitSpew_Codegen, "%s", disasm_buffer.start()); + } + return size; +} + +uintptr_t Assembler::target_address_at(Instruction* pc) { + Instruction* instr0 = pc; + DEBUG_PRINTF("target_address_at: pc: 0x%p\t", instr0); + Instruction* instr1 = pc + 1 * kInstrSize; + Instruction* instr2 = pc + 2 * kInstrSize; + Instruction* instr3 = pc + 3 * kInstrSize; + Instruction* instr4 = pc + 4 * kInstrSize; + Instruction* instr5 = pc + 5 * kInstrSize; + + // Interpret instructions for address generated by li: See listing in + // Assembler::set_target_address_at() just below. + if (IsLui(*reinterpret_cast<Instr*>(instr0)) && + IsAddi(*reinterpret_cast<Instr*>(instr1)) && + IsSlli(*reinterpret_cast<Instr*>(instr2)) && + IsOri(*reinterpret_cast<Instr*>(instr3)) && + IsSlli(*reinterpret_cast<Instr*>(instr4)) && + IsOri(*reinterpret_cast<Instr*>(instr5))) { + // Assemble the 64 bit value. + int64_t addr = (int64_t)(instr0->Imm20UValue() << kImm20Shift) + + (int64_t)instr1->Imm12Value(); + MOZ_ASSERT(instr2->Imm12Value() == 11); + addr <<= 11; + addr |= (int64_t)instr3->Imm12Value(); + MOZ_ASSERT(instr4->Imm12Value() == 6); + addr <<= 6; + addr |= (int64_t)instr5->Imm12Value(); + + DEBUG_PRINTF("addr: %lx\n", addr); + return static_cast<uintptr_t>(addr); + } + // We should never get here, force a bad address if we do. + MOZ_CRASH("RISC-V UNREACHABLE"); +} + +void Assembler::PatchDataWithValueCheck(CodeLocationLabel label, + ImmPtr newValue, ImmPtr expectedValue) { + PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value), + PatchedImmPtr(expectedValue.value)); +} + +void Assembler::PatchDataWithValueCheck(CodeLocationLabel label, + PatchedImmPtr newValue, + PatchedImmPtr expectedValue) { + Instruction* inst = (Instruction*)label.raw(); + + // Extract old Value + DebugOnly<uint64_t> value = Assembler::ExtractLoad64Value(inst); + MOZ_ASSERT(value == uint64_t(expectedValue.value)); + + // Replace with new value + Assembler::UpdateLoad64Value(inst, uint64_t(newValue.value)); +} + +uint64_t Assembler::ExtractLoad64Value(Instruction* inst0) { + DEBUG_PRINTF("\tExtractLoad64Value: \tpc:%p ", inst0); + if (IsJal(*reinterpret_cast<Instr*>(inst0))) { + int offset = inst0->Imm20JValue(); + inst0 = inst0 + offset; + } + Instruction* instr1 = inst0 + 1 * kInstrSize; + if (IsAddiw(*reinterpret_cast<Instr*>(instr1))) { + // Li64 + Instruction* instr2 = inst0 + 2 * kInstrSize; + Instruction* instr3 = inst0 + 3 * kInstrSize; + Instruction* instr4 = inst0 + 4 * kInstrSize; + Instruction* instr5 = inst0 + 5 * kInstrSize; + Instruction* instr6 = inst0 + 6 * kInstrSize; + Instruction* instr7 = inst0 + 7 * kInstrSize; + if (IsLui(*reinterpret_cast<Instr*>(inst0)) && + IsAddiw(*reinterpret_cast<Instr*>(instr1)) && + IsSlli(*reinterpret_cast<Instr*>(instr2)) && + IsAddi(*reinterpret_cast<Instr*>(instr3)) && + IsSlli(*reinterpret_cast<Instr*>(instr4)) && + IsAddi(*reinterpret_cast<Instr*>(instr5)) && + IsSlli(*reinterpret_cast<Instr*>(instr6)) && + IsAddi(*reinterpret_cast<Instr*>(instr7))) { + int64_t imm = (int64_t)(inst0->Imm20UValue() << kImm20Shift) + + (int64_t)instr1->Imm12Value(); + MOZ_ASSERT(instr2->Imm12Value() == 12); + imm <<= 12; + imm += (int64_t)instr3->Imm12Value(); + MOZ_ASSERT(instr4->Imm12Value() == 12); + imm <<= 12; + imm += (int64_t)instr5->Imm12Value(); + MOZ_ASSERT(instr6->Imm12Value() == 12); + imm <<= 12; + imm += (int64_t)instr7->Imm12Value(); + DEBUG_PRINTF("imm:%lx\n", imm); + return imm; + } else { + FLAG_riscv_debug = true; + disassembleInstr(inst0->InstructionBits()); + disassembleInstr(instr1->InstructionBits()); + disassembleInstr(instr2->InstructionBits()); + disassembleInstr(instr3->InstructionBits()); + disassembleInstr(instr4->InstructionBits()); + disassembleInstr(instr5->InstructionBits()); + disassembleInstr(instr6->InstructionBits()); + disassembleInstr(instr7->InstructionBits()); + MOZ_CRASH(); + } + } else { + DEBUG_PRINTF("\n"); + Instruction* instrf1 = (inst0 - 1 * kInstrSize); + Instruction* instr2 = inst0 + 2 * kInstrSize; + Instruction* instr3 = inst0 + 3 * kInstrSize; + Instruction* instr4 = inst0 + 4 * kInstrSize; + Instruction* instr5 = inst0 + 5 * kInstrSize; + Instruction* instr6 = inst0 + 6 * kInstrSize; + Instruction* instr7 = inst0 + 7 * kInstrSize; + disassembleInstr(instrf1->InstructionBits()); + disassembleInstr(inst0->InstructionBits()); + disassembleInstr(instr1->InstructionBits()); + disassembleInstr(instr2->InstructionBits()); + disassembleInstr(instr3->InstructionBits()); + disassembleInstr(instr4->InstructionBits()); + disassembleInstr(instr5->InstructionBits()); + disassembleInstr(instr6->InstructionBits()); + disassembleInstr(instr7->InstructionBits()); + MOZ_ASSERT(IsAddi(*reinterpret_cast<Instr*>(instr1))); + // Li48 + return target_address_at(inst0); + } +} + +void Assembler::UpdateLoad64Value(Instruction* pc, uint64_t value) { + DEBUG_PRINTF("\tUpdateLoad64Value: pc: %p\tvalue: %lx\n", pc, value); + Instruction* instr1 = pc + 1 * kInstrSize; + if (IsJal(*reinterpret_cast<Instr*>(pc))) { + pc = pc + pc->Imm20JValue(); + instr1 = pc + 1 * kInstrSize; + } + if (IsAddiw(*reinterpret_cast<Instr*>(instr1))) { + Instruction* instr0 = pc; + Instruction* instr2 = pc + 2 * kInstrSize; + Instruction* instr3 = pc + 3 * kInstrSize; + Instruction* instr4 = pc + 4 * kInstrSize; + Instruction* instr5 = pc + 5 * kInstrSize; + Instruction* instr6 = pc + 6 * kInstrSize; + Instruction* instr7 = pc + 7 * kInstrSize; + MOZ_ASSERT(IsLui(*reinterpret_cast<Instr*>(pc)) && + IsAddiw(*reinterpret_cast<Instr*>(instr1)) && + IsSlli(*reinterpret_cast<Instr*>(instr2)) && + IsAddi(*reinterpret_cast<Instr*>(instr3)) && + IsSlli(*reinterpret_cast<Instr*>(instr4)) && + IsAddi(*reinterpret_cast<Instr*>(instr5)) && + IsSlli(*reinterpret_cast<Instr*>(instr6)) && + IsAddi(*reinterpret_cast<Instr*>(instr7))); + // lui(rd, (imm + (1LL << 47) + (1LL << 35) + (1LL << 23) + (1LL << 11)) >> + // 48); // Bits 63:48 + // addiw(rd, rd, + // (imm + (1LL << 35) + (1LL << 23) + (1LL << 11)) << 16 >> + // 52); // Bits 47:36 + // slli(rd, rd, 12); + // addi(rd, rd, (imm + (1LL << 23) + (1LL << 11)) << 28 >> 52); // Bits + // 35:24 slli(rd, rd, 12); addi(rd, rd, (imm + (1LL << 11)) << 40 >> 52); // + // Bits 23:12 slli(rd, rd, 12); addi(rd, rd, imm << 52 >> 52); // Bits 11:0 + *reinterpret_cast<Instr*>(instr0) &= 0xfff; + *reinterpret_cast<Instr*>(instr0) |= + (((value + (1LL << 47) + (1LL << 35) + (1LL << 23) + (1LL << 11)) >> 48) + << 12); + *reinterpret_cast<Instr*>(instr1) &= 0xfffff; + *reinterpret_cast<Instr*>(instr1) |= + (((value + (1LL << 35) + (1LL << 23) + (1LL << 11)) << 16 >> 52) << 20); + *reinterpret_cast<Instr*>(instr3) &= 0xfffff; + *reinterpret_cast<Instr*>(instr3) |= + (((value + (1LL << 23) + (1LL << 11)) << 28 >> 52) << 20); + *reinterpret_cast<Instr*>(instr5) &= 0xfffff; + *reinterpret_cast<Instr*>(instr5) |= + (((value + (1LL << 11)) << 40 >> 52) << 20); + *reinterpret_cast<Instr*>(instr7) &= 0xfffff; + *reinterpret_cast<Instr*>(instr7) |= ((value << 52 >> 52) << 20); + disassembleInstr(instr0->InstructionBits()); + disassembleInstr(instr1->InstructionBits()); + disassembleInstr(instr2->InstructionBits()); + disassembleInstr(instr3->InstructionBits()); + disassembleInstr(instr4->InstructionBits()); + disassembleInstr(instr5->InstructionBits()); + disassembleInstr(instr6->InstructionBits()); + disassembleInstr(instr7->InstructionBits()); + MOZ_ASSERT(ExtractLoad64Value(pc) == value); + } else { + Instruction* instr0 = pc; + Instruction* instr2 = pc + 2 * kInstrSize; + Instruction* instr3 = pc + 3 * kInstrSize; + Instruction* instr4 = pc + 4 * kInstrSize; + Instruction* instr5 = pc + 5 * kInstrSize; + Instruction* instr6 = pc + 6 * kInstrSize; + Instruction* instr7 = pc + 7 * kInstrSize; + disassembleInstr(instr0->InstructionBits()); + disassembleInstr(instr1->InstructionBits()); + disassembleInstr(instr2->InstructionBits()); + disassembleInstr(instr3->InstructionBits()); + disassembleInstr(instr4->InstructionBits()); + disassembleInstr(instr5->InstructionBits()); + disassembleInstr(instr6->InstructionBits()); + disassembleInstr(instr7->InstructionBits()); + MOZ_ASSERT(IsAddi(*reinterpret_cast<Instr*>(instr1))); + set_target_value_at(pc, value); + } +} + +void Assembler::set_target_value_at(Instruction* pc, uint64_t target) { + DEBUG_PRINTF("\tset_target_value_at: pc: %p\ttarget: %lx\n", pc, target); + uint32_t* p = reinterpret_cast<uint32_t*>(pc); + MOZ_ASSERT((target & 0xffff000000000000ll) == 0); +#ifdef DEBUG + // Check we have the result from a li macro-instruction. + Instruction* instr0 = pc; + Instruction* instr1 = pc + 1 * kInstrSize; + Instruction* instr3 = pc + 3 * kInstrSize; + Instruction* instr5 = pc + 5 * kInstrSize; + MOZ_ASSERT(IsLui(*reinterpret_cast<Instr*>(instr0)) && + IsAddi(*reinterpret_cast<Instr*>(instr1)) && + IsOri(*reinterpret_cast<Instr*>(instr3)) && + IsOri(*reinterpret_cast<Instr*>(instr5))); +#endif + int64_t a6 = target & 0x3f; // bits 0:6. 6 bits + int64_t b11 = (target >> 6) & 0x7ff; // bits 6:11. 11 bits + int64_t high_31 = (target >> 17) & 0x7fffffff; // 31 bits + int64_t high_20 = ((high_31 + 0x800) >> 12); // 19 bits + int64_t low_12 = high_31 & 0xfff; // 12 bits + *p = *p & 0xfff; + *p = *p | ((int32_t)high_20 << 12); + *(p + 1) = *(p + 1) & 0xfffff; + *(p + 1) = *(p + 1) | ((int32_t)low_12 << 20); + *(p + 2) = *(p + 2) & 0xfffff; + *(p + 2) = *(p + 2) | (11 << 20); + *(p + 3) = *(p + 3) & 0xfffff; + *(p + 3) = *(p + 3) | ((int32_t)b11 << 20); + *(p + 4) = *(p + 4) & 0xfffff; + *(p + 4) = *(p + 4) | (6 << 20); + *(p + 5) = *(p + 5) & 0xfffff; + *(p + 5) = *(p + 5) | ((int32_t)a6 << 20); + MOZ_ASSERT(target_address_at(pc) == target); +} + +void Assembler::WriteLoad64Instructions(Instruction* inst0, Register reg, + uint64_t value) { + DEBUG_PRINTF("\tWriteLoad64Instructions\n"); + // Initialize rd with an address + // Pointers are 48 bits + // 6 fixed instructions are generated + MOZ_ASSERT((value & 0xfff0000000000000ll) == 0); + int64_t a6 = value & 0x3f; // bits 0:5. 6 bits + int64_t b11 = (value >> 6) & 0x7ff; // bits 6:11. 11 bits + int64_t high_31 = (value >> 17) & 0x7fffffff; // 31 bits + int64_t high_20 = ((high_31 + 0x800) >> 12); // 19 bits + int64_t low_12 = high_31 & 0xfff; // 12 bits + Instr lui_ = LUI | (reg.code() << kRdShift) | + ((int32_t)high_20 << kImm20Shift); // lui(rd, (int32_t)high_20); + *reinterpret_cast<Instr*>(inst0) = lui_; + + Instr addi_ = + OP_IMM | (reg.code() << kRdShift) | (0b000 << kFunct3Shift) | + (reg.code() << kRs1Shift) | + (low_12 << kImm12Shift); // addi(rd, rd, low_12); // 31 bits in rd. + *reinterpret_cast<Instr*>(inst0 + 1 * kInstrSize) = addi_; + + Instr slli_ = + OP_IMM | (reg.code() << kRdShift) | (0b001 << kFunct3Shift) | + (reg.code() << kRs1Shift) | + (11 << kImm12Shift); // slli(rd, rd, 11); // Space for next 11 bis + *reinterpret_cast<Instr*>(inst0 + 2 * kInstrSize) = slli_; + + Instr ori_b11 = OP_IMM | (reg.code() << kRdShift) | (0b110 << kFunct3Shift) | + (reg.code() << kRs1Shift) | + (b11 << kImm12Shift); // ori(rd, rd, b11); // 11 bits + // are put in. 42 bit in rd + *reinterpret_cast<Instr*>(inst0 + 3 * kInstrSize) = ori_b11; + + slli_ = OP_IMM | (reg.code() << kRdShift) | (0b001 << kFunct3Shift) | + (reg.code() << kRs1Shift) | + (6 << kImm12Shift); // slli(rd, rd, 6); // Space for next 11 bis + *reinterpret_cast<Instr*>(inst0 + 4 * kInstrSize) = + slli_; // slli(rd, rd, 6); // Space for next 6 bits + + Instr ori_a6 = OP_IMM | (reg.code() << kRdShift) | (0b110 << kFunct3Shift) | + (reg.code() << kRs1Shift) | + (a6 << kImm12Shift); // ori(rd, rd, a6); // 6 bits are + // put in. 48 bis in rd + *reinterpret_cast<Instr*>(inst0 + 5 * kInstrSize) = ori_a6; + disassembleInstr((inst0 + 0 * kInstrSize)->InstructionBits()); + disassembleInstr((inst0 + 1 * kInstrSize)->InstructionBits()); + disassembleInstr((inst0 + 2 * kInstrSize)->InstructionBits()); + disassembleInstr((inst0 + 3 * kInstrSize)->InstructionBits()); + disassembleInstr((inst0 + 4 * kInstrSize)->InstructionBits()); + disassembleInstr((inst0 + 5 * kInstrSize)->InstructionBits()); + disassembleInstr((inst0 + 6 * kInstrSize)->InstructionBits()); + MOZ_ASSERT(ExtractLoad64Value(inst0) == value); +} + +// This just stomps over memory with 32 bits of raw data. Its purpose is to +// overwrite the call of JITed code with 32 bits worth of an offset. This will +// is only meant to function on code that has been invalidated, so it should +// be totally safe. Since that instruction will never be executed again, a +// ICache flush should not be necessary +void Assembler::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) { + // Raw is going to be the return address. + uint32_t* raw = (uint32_t*)label.raw(); + // Overwrite the 4 bytes before the return address, which will + // end up being the call instruction. + *(raw - 1) = imm.value; +} + +void Assembler::target_at_put(BufferOffset pos, BufferOffset target_pos, + bool trampoline) { + if (m_buffer.oom()) { + return; + } + DEBUG_PRINTF("\ttarget_at_put: %p (%d) to %p (%d)\n", + reinterpret_cast<Instr*>(editSrc(pos)), pos.getOffset(), + reinterpret_cast<Instr*>(editSrc(pos)) + target_pos.getOffset() - + pos.getOffset(), + target_pos.getOffset()); + Instruction* instruction = editSrc(pos); + Instr instr = instruction->InstructionBits(); + switch (instruction->InstructionOpcodeType()) { + case BRANCH: { + instr = SetBranchOffset(pos.getOffset(), target_pos.getOffset(), instr); + instr_at_put(pos, instr); + } break; + case JAL: { + MOZ_ASSERT(IsJal(instr)); + instr = SetJalOffset(pos.getOffset(), target_pos.getOffset(), instr); + instr_at_put(pos, instr); + } break; + case LUI: { + set_target_value_at(instruction, + reinterpret_cast<uintptr_t>(editSrc(target_pos))); + } break; + case AUIPC: { + Instr instr_auipc = instr; + Instr instr_I = + editSrc(BufferOffset(pos.getOffset() + 4))->InstructionBits(); + MOZ_ASSERT(IsJalr(instr_I) || IsAddi(instr_I)); + + intptr_t offset = target_pos.getOffset() - pos.getOffset(); + if (is_int21(offset) && IsJalr(instr_I) && trampoline) { + MOZ_ASSERT(is_int21(offset) && ((offset & 1) == 0)); + Instr instr = JAL; + instr = SetJalOffset(pos.getOffset(), target_pos.getOffset(), instr); + MOZ_ASSERT(IsJal(instr)); + MOZ_ASSERT(JumpOffset(instr) == offset); + instr_at_put(pos, instr); + instr_at_put(BufferOffset(pos.getOffset() + 4), kNopByte); + } else { + MOZ_RELEASE_ASSERT(is_int32(offset + 0x800)); + MOZ_ASSERT(instruction->RdValue() == + editSrc(BufferOffset(pos.getOffset() + 4))->Rs1Value()); + int32_t Hi20 = (((int32_t)offset + 0x800) >> 12); + int32_t Lo12 = (int32_t)offset << 20 >> 20; + + instr_auipc = + (instr_auipc & ~kImm31_12Mask) | ((Hi20 & kImm19_0Mask) << 12); + instr_at_put(pos, instr_auipc); + + const int kImm31_20Mask = ((1 << 12) - 1) << 20; + const int kImm11_0Mask = ((1 << 12) - 1); + instr_I = (instr_I & ~kImm31_20Mask) | ((Lo12 & kImm11_0Mask) << 20); + instr_at_put(BufferOffset(pos.getOffset() + 4), instr_I); + } + } break; + default: + UNIMPLEMENTED_RISCV(); + break; + } +} + +const int kEndOfChain = -1; +const int32_t kEndOfJumpChain = 0; + +int Assembler::target_at(BufferOffset pos, bool is_internal) { + if (oom()) { + return kEndOfChain; + } + Instruction* instruction = editSrc(pos); + Instruction* instruction2 = nullptr; + if (IsAuipc(instruction->InstructionBits())) { + instruction2 = editSrc(BufferOffset(pos.getOffset() + kInstrSize)); + } + return target_at(instruction, pos, is_internal, instruction2); +} + +int Assembler::target_at(Instruction* instruction, BufferOffset pos, + bool is_internal, Instruction* instruction2) { + DEBUG_PRINTF("\t target_at: %p(%x)\n\t", + reinterpret_cast<Instr*>(instruction), pos.getOffset()); + disassembleInstr(instruction->InstructionBits()); + Instr instr = instruction->InstructionBits(); + switch (instruction->InstructionOpcodeType()) { + case BRANCH: { + int32_t imm13 = BranchOffset(instr); + if (imm13 == kEndOfJumpChain) { + // EndOfChain sentinel is returned directly, not relative to pc or pos. + return kEndOfChain; + } else { + DEBUG_PRINTF("\t target_at: %d %d\n", imm13, pos.getOffset() + imm13); + return pos.getOffset() + imm13; + } + } + case JAL: { + int32_t imm21 = JumpOffset(instr); + if (imm21 == kEndOfJumpChain) { + // EndOfChain sentinel is returned directly, not relative to pc or pos. + return kEndOfChain; + } else { + DEBUG_PRINTF("\t target_at: %d %d\n", imm21, pos.getOffset() + imm21); + return pos.getOffset() + imm21; + } + } + case JALR: { + int32_t imm12 = instr >> 20; + if (imm12 == kEndOfJumpChain) { + // EndOfChain sentinel is returned directly, not relative to pc or pos. + return kEndOfChain; + } else { + DEBUG_PRINTF("\t target_at: %d %d\n", imm12, pos.getOffset() + imm12); + return pos.getOffset() + imm12; + } + } + case LUI: { + uintptr_t imm = target_address_at(instruction); + uintptr_t instr_address = reinterpret_cast<uintptr_t>(instruction); + if (imm == kEndOfJumpChain) { + return kEndOfChain; + } else { + MOZ_ASSERT(instr_address - imm < INT_MAX); + int32_t delta = static_cast<int32_t>(instr_address - imm); + MOZ_ASSERT(pos.getOffset() > delta); + return pos.getOffset() - delta; + } + } + case AUIPC: { + MOZ_ASSERT(instruction2 != nullptr); + Instr instr_auipc = instr; + Instr instr_I = instruction2->InstructionBits(); + MOZ_ASSERT(IsJalr(instr_I) || IsAddi(instr_I)); + int32_t offset = BrachlongOffset(instr_auipc, instr_I); + if (offset == kEndOfJumpChain) return kEndOfChain; + DEBUG_PRINTF("\t target_at: %d %d\n", offset, pos.getOffset() + offset); + return offset + pos.getOffset(); + } + default: { + UNIMPLEMENTED_RISCV(); + } + } +} + +uint32_t Assembler::next_link(Label* L, bool is_internal) { + MOZ_ASSERT(L->used()); + BufferOffset pos(L); + int link = target_at(pos, is_internal); + if (link == kEndOfChain) { + L->reset(); + return LabelBase::INVALID_OFFSET; + } else { + MOZ_ASSERT(link >= 0); + DEBUG_PRINTF("next: %p to offset %d\n", L, link); + L->use(link); + return link; + } +} + +void Assembler::bind(Label* label, BufferOffset boff) { + JitSpew(JitSpew_Codegen, ".set Llabel %p %d", label, currentOffset()); + DEBUG_PRINTF(".set Llabel %p\n", label); + // If our caller didn't give us an explicit target to bind to + // then we want to bind to the location of the next instruction + BufferOffset dest = boff.assigned() ? boff : nextOffset(); + if (label->used()) { + uint32_t next; + + // A used label holds a link to branch that uses it. + do { + BufferOffset b(label); + DEBUG_PRINTF("\tbind next:%d\n", b.getOffset()); + // Even a 0 offset may be invalid if we're out of memory. + if (oom()) { + return; + } + int fixup_pos = b.getOffset(); + int dist = dest.getOffset() - fixup_pos; + next = next_link(label, false); + DEBUG_PRINTF("\t%p fixup: %d next: %d\n", label, fixup_pos, next); + DEBUG_PRINTF("\t fixup: %d dest: %d dist: %d %d %d\n", fixup_pos, + dest.getOffset(), dist, nextOffset().getOffset(), + currentOffset()); + Instruction* instruction = editSrc(b); + Instr instr = instruction->InstructionBits(); + if (IsBranch(instr)) { + if (dist > kMaxBranchOffset) { + MOZ_ASSERT(next != LabelBase::INVALID_OFFSET); + MOZ_RELEASE_ASSERT((next - fixup_pos) <= kMaxBranchOffset); + MOZ_ASSERT(IsAuipc(editSrc(BufferOffset(next))->InstructionBits())); + MOZ_ASSERT( + IsJalr(editSrc(BufferOffset(next + 4))->InstructionBits())); + DEBUG_PRINTF("\t\ttrampolining: %d\n", next); + } else { + target_at_put(b, dest); + BufferOffset deadline(b.getOffset() + + ImmBranchMaxForwardOffset(CondBranchRangeType)); + m_buffer.unregisterBranchDeadline(CondBranchRangeType, deadline); + } + } else if (IsJal(instr)) { + if (dist > kMaxJumpOffset) { + MOZ_ASSERT(next != LabelBase::INVALID_OFFSET); + MOZ_RELEASE_ASSERT((next - fixup_pos) <= kMaxJumpOffset); + MOZ_ASSERT(IsAuipc(editSrc(BufferOffset(next))->InstructionBits())); + MOZ_ASSERT( + IsJalr(editSrc(BufferOffset(next + 4))->InstructionBits())); + DEBUG_PRINTF("\t\ttrampolining: %d\n", next); + } else { + target_at_put(b, dest); + BufferOffset deadline( + b.getOffset() + ImmBranchMaxForwardOffset(UncondBranchRangeType)); + m_buffer.unregisterBranchDeadline(UncondBranchRangeType, deadline); + } + } else { + MOZ_ASSERT(IsAuipc(instr)); + target_at_put(b, dest); + } + } while (next != LabelBase::INVALID_OFFSET); + } + label->bind(dest.getOffset()); +} + +void Assembler::Bind(uint8_t* rawCode, const CodeLabel& label) { + if (label.patchAt().bound()) { + auto mode = label.linkMode(); + intptr_t offset = label.patchAt().offset(); + intptr_t target = label.target().offset(); + + if (mode == CodeLabel::RawPointer) { + *reinterpret_cast<const void**>(rawCode + offset) = rawCode + target; + } else { + MOZ_ASSERT(mode == CodeLabel::MoveImmediate || + mode == CodeLabel::JumpImmediate); + Instruction* inst = (Instruction*)(rawCode + offset); + Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + target)); + } + } +} + +bool Assembler::is_near(Label* L) { + MOZ_ASSERT(L->bound()); + return is_intn((currentOffset() - L->offset()), kJumpOffsetBits); +} + +bool Assembler::is_near(Label* L, OffsetSize bits) { + if (L == nullptr || !L->bound()) return true; + return is_intn((currentOffset() - L->offset()), bits); +} + +bool Assembler::is_near_branch(Label* L) { + MOZ_ASSERT(L->bound()); + return is_intn((currentOffset() - L->offset()), kBranchOffsetBits); +} + +int32_t Assembler::branch_long_offset(Label* L) { + if (oom()) { + return kEndOfJumpChain; + } + intptr_t target_pos; + BufferOffset next_instr_offset = nextInstrOffset(2); + DEBUG_PRINTF("\tbranch_long_offset: %p to (%d)\n", L, + next_instr_offset.getOffset()); + if (L->bound()) { + JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L, + next_instr_offset.getOffset()); + target_pos = L->offset(); + } else { + if (L->used()) { + LabelCahe::Ptr p = label_cache_.lookup(L->offset()); + MOZ_ASSERT(p); + MOZ_ASSERT(p->key() == L->offset()); + target_pos = p->value().getOffset(); + target_at_put(BufferOffset(target_pos), next_instr_offset); + DEBUG_PRINTF("\tLabel %p added to link: %d\n", L, + next_instr_offset.getOffset()); + bool ok = label_cache_.put(L->offset(), next_instr_offset); + if (!ok) { + NoEnoughLabelCache(); + } + return kEndOfJumpChain; + } else { + JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L, + next_instr_offset.getOffset()); + L->use(next_instr_offset.getOffset()); + DEBUG_PRINTF("\tLabel %p added to link: %d\n", L, + next_instr_offset.getOffset()); + bool ok = label_cache_.putNew(L->offset(), next_instr_offset); + if (!ok) { + NoEnoughLabelCache(); + } + return kEndOfJumpChain; + } + } + intptr_t offset = target_pos - next_instr_offset.getOffset(); + MOZ_ASSERT((offset & 3) == 0); + MOZ_ASSERT(is_int32(offset)); + return static_cast<int32_t>(offset); +} + +int32_t Assembler::branch_offset_helper(Label* L, OffsetSize bits) { + if (oom()) { + return kEndOfJumpChain; + } + int32_t target_pos; + BufferOffset next_instr_offset = nextInstrOffset(); + DEBUG_PRINTF("\tbranch_offset_helper: %p to %d\n", L, + next_instr_offset.getOffset()); + // This is the last possible branch target. + if (L->bound()) { + JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L, + next_instr_offset.getOffset()); + target_pos = L->offset(); + } else { + BufferOffset deadline(next_instr_offset.getOffset() + + ImmBranchMaxForwardOffset(bits)); + DEBUG_PRINTF("\tregisterBranchDeadline %d type %d\n", deadline.getOffset(), + OffsetSizeToImmBranchRangeType(bits)); + m_buffer.registerBranchDeadline(OffsetSizeToImmBranchRangeType(bits), + deadline); + if (L->used()) { + LabelCahe::Ptr p = label_cache_.lookup(L->offset()); + MOZ_ASSERT(p); + MOZ_ASSERT(p->key() == L->offset()); + target_pos = p->value().getOffset(); + target_at_put(BufferOffset(target_pos), next_instr_offset); + DEBUG_PRINTF("\tLabel %p added to link: %d\n", L, + next_instr_offset.getOffset()); + bool ok = label_cache_.put(L->offset(), next_instr_offset); + if (!ok) { + NoEnoughLabelCache(); + } + return kEndOfJumpChain; + } else { + JitSpew(JitSpew_Codegen, ".use Llabel %p on %d", L, + next_instr_offset.getOffset()); + L->use(next_instr_offset.getOffset()); + bool ok = label_cache_.putNew(L->offset(), next_instr_offset); + if (!ok) { + NoEnoughLabelCache(); + } + DEBUG_PRINTF("\tLabel %p added to link: %d\n", L, + next_instr_offset.getOffset()); + return kEndOfJumpChain; + } + } + + int32_t offset = target_pos - next_instr_offset.getOffset(); + DEBUG_PRINTF("\toffset = %d\n", offset); + MOZ_ASSERT(is_intn(offset, bits)); + MOZ_ASSERT((offset & 1) == 0); + return offset; +} + +Assembler::Condition Assembler::InvertCondition(Condition cond) { + switch (cond) { + case Equal: + return NotEqual; + case NotEqual: + return Equal; + case Zero: + return NonZero; + case NonZero: + return Zero; + case LessThan: + return GreaterThanOrEqual; + case LessThanOrEqual: + return GreaterThan; + case GreaterThan: + return LessThanOrEqual; + case GreaterThanOrEqual: + return LessThan; + case Above: + return BelowOrEqual; + case AboveOrEqual: + return Below; + case Below: + return AboveOrEqual; + case BelowOrEqual: + return Above; + case Signed: + return NotSigned; + case NotSigned: + return Signed; + default: + MOZ_CRASH("unexpected condition"); + } +} + +Assembler::DoubleCondition Assembler::InvertCondition(DoubleCondition cond) { + switch (cond) { + case DoubleOrdered: + return DoubleUnordered; + case DoubleEqual: + return DoubleNotEqualOrUnordered; + case DoubleNotEqual: + return DoubleEqualOrUnordered; + case DoubleGreaterThan: + return DoubleLessThanOrEqualOrUnordered; + case DoubleGreaterThanOrEqual: + return DoubleLessThanOrUnordered; + case DoubleLessThan: + return DoubleGreaterThanOrEqualOrUnordered; + case DoubleLessThanOrEqual: + return DoubleGreaterThanOrUnordered; + case DoubleUnordered: + return DoubleOrdered; + case DoubleEqualOrUnordered: + return DoubleNotEqual; + case DoubleNotEqualOrUnordered: + return DoubleEqual; + case DoubleGreaterThanOrUnordered: + return DoubleLessThanOrEqual; + case DoubleGreaterThanOrEqualOrUnordered: + return DoubleLessThan; + case DoubleLessThanOrUnordered: + return DoubleGreaterThanOrEqual; + case DoubleLessThanOrEqualOrUnordered: + return DoubleGreaterThan; + default: + MOZ_CRASH("unexpected condition"); + } +} + +// Break / Trap instructions. +void Assembler::break_(uint32_t code, bool break_as_stop) { + // We need to invalidate breaks that could be stops as well because the + // simulator expects a char pointer after the stop instruction. + // See constants-mips.h for explanation. + MOZ_ASSERT( + (break_as_stop && code <= kMaxStopCode && code > kMaxTracepointCode) || + (!break_as_stop && (code > kMaxStopCode || code <= kMaxTracepointCode))); + + // since ebreak does not allow additional immediate field, we use the + // immediate field of lui instruction immediately following the ebreak to + // encode the "code" info + ebreak(); + MOZ_ASSERT(is_uint20(code)); + lui(zero_reg, code); +} + +void Assembler::ToggleToJmp(CodeLocationLabel inst_) { + Instruction* inst = (Instruction*)inst_.raw(); + MOZ_ASSERT(IsAddi(inst->InstructionBits())); + int32_t offset = inst->Imm12Value(); + MOZ_ASSERT(is_int12(offset)); + Instr jal_ = JAL | (0b000 << kFunct3Shift) | + (offset & 0xff000) | // bits 19-12 + ((offset & 0x800) << 9) | // bit 11 + ((offset & 0x7fe) << 20) | // bits 10-1 + ((offset & 0x100000) << 11); // bit 20 + // jal(zero, offset); + *reinterpret_cast<Instr*>(inst) = jal_; +} + +void Assembler::ToggleToCmp(CodeLocationLabel inst_) { + Instruction* inst = (Instruction*)inst_.raw(); + + // toggledJump is allways used for short jumps. + MOZ_ASSERT(IsJal(inst->InstructionBits())); + // Replace "jal zero_reg, offset" with "addi $zero, $zero, offset" + int32_t offset = inst->Imm20JValue(); + MOZ_ASSERT(is_int12(offset)); + Instr addi_ = OP_IMM | (0b000 << kFunct3Shift) | + (offset << kImm12Shift); // addi(zero, zero, low_12); + *reinterpret_cast<Instr*>(inst) = addi_; +} + +bool Assembler::reserve(size_t size) { + // This buffer uses fixed-size chunks so there's no point in reserving + // now vs. on-demand. + return !oom(); +} + +static JitCode* CodeFromJump(Instruction* jump) { + uint8_t* target = (uint8_t*)Assembler::ExtractLoad64Value(jump); + return JitCode::FromExecutable(target); +} + +void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + while (reader.more()) { + JitCode* child = + CodeFromJump((Instruction*)(code->raw() + reader.readUnsigned())); + TraceManuallyBarrieredEdge(trc, &child, "rel32"); + } +} + +static void TraceOneDataRelocation(JSTracer* trc, + mozilla::Maybe<AutoWritableJitCode>& awjc, + JitCode* code, Instruction* inst) { + void* ptr = (void*)Assembler::ExtractLoad64Value(inst); + void* prior = ptr; + + // Data relocations can be for Values or for raw pointers. If a Value is + // zero-tagged, we can trace it as if it were a raw pointer. If a Value + // is not zero-tagged, we have to interpret it as a Value to ensure that the + // tag bits are masked off to recover the actual pointer. + uintptr_t word = reinterpret_cast<uintptr_t>(ptr); + if (word >> JSVAL_TAG_SHIFT) { + // This relocation is a Value with a non-zero tag. + Value v = Value::fromRawBits(word); + TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value"); + ptr = (void*)v.bitsAsPunboxPointer(); + } else { + // This relocation is a raw pointer or a Value with a zero tag. + // No barrier needed since these are constants. + TraceManuallyBarrieredGenericPointerEdge( + trc, reinterpret_cast<gc::Cell**>(&ptr), "jit-masm-ptr"); + } + + if (ptr != prior) { + if (awjc.isNothing()) { + awjc.emplace(code); + } + Assembler::UpdateLoad64Value(inst, uint64_t(ptr)); + } +} + +/* static */ +void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code, + CompactBufferReader& reader) { + mozilla::Maybe<AutoWritableJitCode> awjc; + while (reader.more()) { + size_t offset = reader.readUnsigned(); + Instruction* inst = (Instruction*)(code->raw() + offset); + TraceOneDataRelocation(trc, awjc, code, inst); + } +} + +UseScratchRegisterScope::UseScratchRegisterScope(Assembler* assembler) + : available_(assembler->GetScratchRegisterList()), + old_available_(*available_) {} + +UseScratchRegisterScope::~UseScratchRegisterScope() { + *available_ = old_available_; +} + +Register UseScratchRegisterScope::Acquire() { + MOZ_ASSERT(available_ != nullptr); + MOZ_ASSERT(!available_->empty()); + Register index = GeneralRegisterSet::FirstRegister(available_->bits()); + available_->takeRegisterIndex(index); + return index; +} + +bool UseScratchRegisterScope::hasAvailable() const { + return (available_->size()) != 0; +} + +void Assembler::retarget(Label* label, Label* target) { + spew("retarget %p -> %p", label, target); + if (label->used() && !oom()) { + if (target->bound()) { + bind(label, BufferOffset(target)); + } else if (target->used()) { + // The target is not bound but used. Prepend label's branch list + // onto target's. + int32_t next; + BufferOffset labelBranchOffset(label); + + // Find the head of the use chain for label. + do { + next = next_link(label, false); + labelBranchOffset = BufferOffset(next); + } while (next != LabelBase::INVALID_OFFSET); + + // Then patch the head of label's use chain to the tail of + // target's use chain, prepending the entire use chain of target. + target->use(label->offset()); + target_at_put(labelBranchOffset, BufferOffset(target)); + MOZ_CRASH("check"); + } else { + // The target is unbound and unused. We can just take the head of + // the list hanging off of label, and dump that into target. + target->use(label->offset()); + } + } + label->reset(); +} + +bool Assembler::appendRawCode(const uint8_t* code, size_t numBytes) { + if (m_buffer.oom()) { + return false; + } + while (numBytes > SliceSize) { + m_buffer.putBytes(SliceSize, code); + numBytes -= SliceSize; + code += SliceSize; + } + m_buffer.putBytes(numBytes, code); + return !m_buffer.oom(); +} + +void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) { + Instruction* i0 = (Instruction*)inst_.raw(); + Instruction* i1 = (Instruction*)(inst_.raw() + 1 * kInstrSize); + Instruction* i2 = (Instruction*)(inst_.raw() + 2 * kInstrSize); + Instruction* i3 = (Instruction*)(inst_.raw() + 3 * kInstrSize); + Instruction* i4 = (Instruction*)(inst_.raw() + 4 * kInstrSize); + Instruction* i5 = (Instruction*)(inst_.raw() + 5 * kInstrSize); + Instruction* i6 = (Instruction*)(inst_.raw() + 6 * kInstrSize); + + MOZ_ASSERT(IsLui(i0->InstructionBits())); + MOZ_ASSERT(IsAddi(i1->InstructionBits())); + MOZ_ASSERT(IsSlli(i2->InstructionBits())); + MOZ_ASSERT(IsOri(i3->InstructionBits())); + MOZ_ASSERT(IsSlli(i4->InstructionBits())); + MOZ_ASSERT(IsOri(i5->InstructionBits())); + if (enabled) { + Instr jalr_ = JALR | (ra.code() << kRdShift) | (0x0 << kFunct3Shift) | + (i5->RdValue() << kRs1Shift) | (0x0 << kImm12Shift); + *((Instr*)i6) = jalr_; + } else { + *((Instr*)i6) = kNopByte; + } +} + +void Assembler::PatchShortRangeBranchToVeneer(Buffer* buffer, unsigned rangeIdx, + BufferOffset deadline, + BufferOffset veneer) { + if (buffer->oom()) { + return; + } + DEBUG_PRINTF("\tPatchShortRangeBranchToVeneer\n"); + // Reconstruct the position of the branch from (rangeIdx, deadline). + ImmBranchRangeType branchRange = static_cast<ImmBranchRangeType>(rangeIdx); + BufferOffset branch(deadline.getOffset() - + ImmBranchMaxForwardOffset(branchRange)); + Instruction* branchInst = buffer->getInst(branch); + Instruction* veneerInst_1 = buffer->getInst(veneer); + Instruction* veneerInst_2 = + buffer->getInst(BufferOffset(veneer.getOffset() + 4)); + // Verify that the branch range matches what's encoded. + DEBUG_PRINTF("\t%p(%x): ", branchInst, branch.getOffset()); + disassembleInstr(branchInst->InstructionBits(), JitSpew_Codegen); + DEBUG_PRINTF("\t instert veneer %x, branch:%x deadline: %x\n", + veneer.getOffset(), branch.getOffset(), deadline.getOffset()); + MOZ_ASSERT(branchRange <= UncondBranchRangeType); + MOZ_ASSERT(branchInst->GetImmBranchRangeType() == branchRange); + // emit a long jump slot + Instr auipc = AUIPC | (t6.code() << kRdShift) | (0x0 << kImm20Shift); + Instr jalr = JALR | (zero_reg.code() << kRdShift) | (0x0 << kFunct3Shift) | + (t6.code() << kRs1Shift) | (0x0 << kImm12Shift); + + // We want to insert veneer after branch in the linked list of instructions + // that use the same unbound label. + // The veneer should be an unconditional branch. + int32_t nextElemOffset = target_at(buffer->getInst(branch), branch, false); + int32_t dist; + // If offset is 0, this is the end of the linked list. + if (nextElemOffset != kEndOfChain) { + // Make the offset relative to veneer so it targets the same instruction + // as branchInst. + dist = nextElemOffset - veneer.getOffset(); + } else { + dist = 0; + } + int32_t Hi20 = (((int32_t)dist + 0x800) >> 12); + int32_t Lo12 = (int32_t)dist << 20 >> 20; + auipc = SetAuipcOffset(Hi20, auipc); + jalr = SetJalrOffset(Lo12, jalr); + // insert veneer + veneerInst_1->SetInstructionBits(auipc); + veneerInst_2->SetInstructionBits(jalr); + // Now link branchInst to veneer. + if (IsBranch(branchInst->InstructionBits())) { + branchInst->SetInstructionBits(SetBranchOffset( + branch.getOffset(), veneer.getOffset(), branchInst->InstructionBits())); + } else { + MOZ_ASSERT(IsJal(branchInst->InstructionBits())); + branchInst->SetInstructionBits(SetJalOffset( + branch.getOffset(), veneer.getOffset(), branchInst->InstructionBits())); + } + DEBUG_PRINTF("\tfix to veneer:"); + disassembleInstr(branchInst->InstructionBits()); +} +} // namespace jit +} // namespace js |