diff options
Diffstat (limited to 'js/src/jit/RegisterAllocator.cpp')
| -rw-r--r-- | js/src/jit/RegisterAllocator.cpp | 669 |
1 files changed, 669 insertions, 0 deletions
diff --git a/js/src/jit/RegisterAllocator.cpp b/js/src/jit/RegisterAllocator.cpp new file mode 100644 index 0000000000..c572e4bc2a --- /dev/null +++ b/js/src/jit/RegisterAllocator.cpp @@ -0,0 +1,669 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/RegisterAllocator.h" + +using namespace js; +using namespace js::jit; + +#ifdef DEBUG +bool AllocationIntegrityState::record() { + // Ignore repeated record() calls. + if (!instructions.empty()) { + return true; + } + + if (!instructions.appendN(InstructionInfo(), graph.numInstructions())) { + return false; + } + + if (!virtualRegisters.appendN((LDefinition*)nullptr, + graph.numVirtualRegisters())) { + return false; + } + + if (!blocks.reserve(graph.numBlocks())) { + return false; + } + for (size_t i = 0; i < graph.numBlocks(); i++) { + blocks.infallibleAppend(BlockInfo()); + LBlock* block = graph.getBlock(i); + MOZ_ASSERT(block->mir()->id() == i); + + BlockInfo& blockInfo = blocks[i]; + if (!blockInfo.phis.reserve(block->numPhis())) { + return false; + } + + for (size_t j = 0; j < block->numPhis(); j++) { + blockInfo.phis.infallibleAppend(InstructionInfo()); + InstructionInfo& info = blockInfo.phis[j]; + LPhi* phi = block->getPhi(j); + MOZ_ASSERT(phi->numDefs() == 1); + uint32_t vreg = phi->getDef(0)->virtualRegister(); + virtualRegisters[vreg] = phi->getDef(0); + if (!info.outputs.append(*phi->getDef(0))) { + return false; + } + for (size_t k = 0, kend = phi->numOperands(); k < kend; k++) { + if (!info.inputs.append(*phi->getOperand(k))) { + return false; + } + } + } + + for (LInstructionIterator iter = block->begin(); iter != block->end(); + iter++) { + LInstruction* ins = *iter; + InstructionInfo& info = instructions[ins->id()]; + + for (size_t k = 0; k < ins->numTemps(); k++) { + if (!ins->getTemp(k)->isBogusTemp()) { + uint32_t vreg = ins->getTemp(k)->virtualRegister(); + virtualRegisters[vreg] = ins->getTemp(k); + } + if (!info.temps.append(*ins->getTemp(k))) { + return false; + } + } + for (size_t k = 0; k < ins->numDefs(); k++) { + if (!ins->getDef(k)->isBogusTemp()) { + uint32_t vreg = ins->getDef(k)->virtualRegister(); + virtualRegisters[vreg] = ins->getDef(k); + } + if (!info.outputs.append(*ins->getDef(k))) { + return false; + } + } + for (LInstruction::InputIterator alloc(*ins); alloc.more(); + alloc.next()) { + if (!info.inputs.append(**alloc)) { + return false; + } + } + } + } + + return true; +} + +bool AllocationIntegrityState::check() { + MOZ_ASSERT(!instructions.empty()); + +# ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_RegAlloc)) { + dump(); + } +# endif + for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) { + LBlock* block = graph.getBlock(blockIndex); + + // Check that all instruction inputs and outputs have been assigned an + // allocation. + for (LInstructionIterator iter = block->begin(); iter != block->end(); + iter++) { + LInstruction* ins = *iter; + + for (LInstruction::InputIterator alloc(*ins); alloc.more(); + alloc.next()) { + MOZ_ASSERT(!alloc->isUse()); + } + + for (size_t i = 0; i < ins->numDefs(); i++) { + LDefinition* def = ins->getDef(i); + MOZ_ASSERT(!def->output()->isUse()); + + LDefinition oldDef = instructions[ins->id()].outputs[i]; + MOZ_ASSERT_IF( + oldDef.policy() == LDefinition::MUST_REUSE_INPUT, + *def->output() == *ins->getOperand(oldDef.getReusedInput())); + } + + for (size_t i = 0; i < ins->numTemps(); i++) { + LDefinition* temp = ins->getTemp(i); + MOZ_ASSERT_IF(!temp->isBogusTemp(), temp->output()->isRegister()); + + LDefinition oldTemp = instructions[ins->id()].temps[i]; + MOZ_ASSERT_IF( + oldTemp.policy() == LDefinition::MUST_REUSE_INPUT, + *temp->output() == *ins->getOperand(oldTemp.getReusedInput())); + } + } + } + + // Check that the register assignment and move groups preserve the original + // semantics of the virtual registers. Each virtual register has a single + // write (owing to the SSA representation), but the allocation may move the + // written value around between registers and memory locations along + // different paths through the script. + // + // For each use of an allocation, follow the physical value which is read + // backward through the script, along all paths to the value's virtual + // register's definition. + for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) { + LBlock* block = graph.getBlock(blockIndex); + for (LInstructionIterator iter = block->begin(); iter != block->end(); + iter++) { + LInstruction* ins = *iter; + const InstructionInfo& info = instructions[ins->id()]; + + LSafepoint* safepoint = ins->safepoint(); + if (safepoint) { + for (size_t i = 0; i < ins->numTemps(); i++) { + if (ins->getTemp(i)->isBogusTemp()) { + continue; + } + uint32_t vreg = info.temps[i].virtualRegister(); + LAllocation* alloc = ins->getTemp(i)->output(); + checkSafepointAllocation(ins, vreg, *alloc); + } + MOZ_ASSERT_IF(ins->isCall(), safepoint->liveRegs().emptyFloat() && + safepoint->liveRegs().emptyGeneral()); + } + + size_t inputIndex = 0; + for (LInstruction::InputIterator alloc(*ins); alloc.more(); + inputIndex++, alloc.next()) { + LAllocation oldInput = info.inputs[inputIndex]; + if (!oldInput.isUse()) { + continue; + } + + uint32_t vreg = oldInput.toUse()->virtualRegister(); + + if (safepoint && !oldInput.toUse()->usedAtStart()) { + checkSafepointAllocation(ins, vreg, **alloc); + } + + // Temps must never alias inputs (even at-start uses) unless explicitly + // requested. + for (size_t i = 0; i < ins->numTemps(); i++) { + if (ins->getTemp(i)->isBogusTemp()) { + continue; + } + LAllocation* tempAlloc = ins->getTemp(i)->output(); + + // Fixed uses and fixed temps are allowed to alias. + if (oldInput.toUse()->isFixedRegister() && info.temps[i].isFixed()) { + continue; + } + + // MUST_REUSE_INPUT temps will alias their input. + if (info.temps[i].policy() == LDefinition::MUST_REUSE_INPUT && + info.temps[i].getReusedInput() == inputIndex) { + continue; + } + + MOZ_ASSERT(!tempAlloc->aliases(**alloc)); + } + + // Start checking at the previous instruction, in case this + // instruction reuses its input register for an output. + LInstructionReverseIterator riter = block->rbegin(ins); + riter++; + if (!checkIntegrity(block, *riter, vreg, **alloc)) { + return false; + } + + while (!worklist.empty()) { + IntegrityItem item = worklist.popCopy(); + if (!checkIntegrity(item.block, *item.block->rbegin(), item.vreg, + item.alloc)) { + return false; + } + } + } + } + } + + return true; +} + +bool AllocationIntegrityState::checkIntegrity(LBlock* block, LInstruction* ins, + uint32_t vreg, + LAllocation alloc) { + for (LInstructionReverseIterator iter(block->rbegin(ins)); + iter != block->rend(); iter++) { + ins = *iter; + + // Follow values through assignments in move groups. All assignments in + // a move group are considered to happen simultaneously, so stop after + // the first matching move is found. + if (ins->isMoveGroup()) { + LMoveGroup* group = ins->toMoveGroup(); + for (int i = group->numMoves() - 1; i >= 0; i--) { + if (group->getMove(i).to() == alloc) { + alloc = group->getMove(i).from(); + break; + } + } + } + + const InstructionInfo& info = instructions[ins->id()]; + + // Make sure the physical location being tracked is not clobbered by + // another instruction, and that if the originating vreg definition is + // found that it is writing to the tracked location. + + for (size_t i = 0; i < ins->numDefs(); i++) { + LDefinition* def = ins->getDef(i); + if (def->isBogusTemp()) { + continue; + } + if (info.outputs[i].virtualRegister() == vreg) { +# ifdef JS_JITSPEW + // If the following assertion is about to fail, print some useful info. + if (!(*def->output() == alloc) && JitSpewEnabled(JitSpew_RegAlloc)) { + CodePosition input(ins->id(), CodePosition::INPUT); + CodePosition output(ins->id(), CodePosition::OUTPUT); + JitSpew(JitSpew_RegAlloc, + "Instruction at %u-%u, output number %u:", input.bits(), + output.bits(), unsigned(i)); + JitSpew(JitSpew_RegAlloc, + " Error: conflicting allocations: %s vs %s", + (*def->output()).toString().get(), alloc.toString().get()); + } +# endif + MOZ_ASSERT(*def->output() == alloc); + + // Found the original definition, done scanning. + return true; + } else { + MOZ_ASSERT(*def->output() != alloc); + } + } + + for (size_t i = 0; i < ins->numTemps(); i++) { + LDefinition* temp = ins->getTemp(i); + if (!temp->isBogusTemp()) { + MOZ_ASSERT(*temp->output() != alloc); + } + } + + if (ins->safepoint()) { + checkSafepointAllocation(ins, vreg, alloc); + } + } + + // Phis are effectless, but change the vreg we are tracking. Check if there + // is one which produced this vreg. We need to follow back through the phi + // inputs as it is not guaranteed the register allocator filled in physical + // allocations for the inputs and outputs of the phis. + for (size_t i = 0; i < block->numPhis(); i++) { + const InstructionInfo& info = blocks[block->mir()->id()].phis[i]; + LPhi* phi = block->getPhi(i); + if (info.outputs[0].virtualRegister() == vreg) { + for (size_t j = 0, jend = phi->numOperands(); j < jend; j++) { + uint32_t newvreg = info.inputs[j].toUse()->virtualRegister(); + LBlock* predecessor = block->mir()->getPredecessor(j)->lir(); + if (!addPredecessor(predecessor, newvreg, alloc)) { + return false; + } + } + return true; + } + } + + // No phi which defined the vreg we are tracking, follow back through all + // predecessors with the existing vreg. + for (size_t i = 0, iend = block->mir()->numPredecessors(); i < iend; i++) { + LBlock* predecessor = block->mir()->getPredecessor(i)->lir(); + if (!addPredecessor(predecessor, vreg, alloc)) { + return false; + } + } + + return true; +} + +void AllocationIntegrityState::checkSafepointAllocation(LInstruction* ins, + uint32_t vreg, + LAllocation alloc) { + LSafepoint* safepoint = ins->safepoint(); + MOZ_ASSERT(safepoint); + + if (ins->isCall() && alloc.isRegister()) { + return; + } + + if (alloc.isRegister()) { + MOZ_ASSERT(safepoint->liveRegs().has(alloc.toRegister())); + } + + // The |this| argument slot is implicitly included in all safepoints. + if (alloc.isArgument() && + alloc.toArgument()->index() < THIS_FRAME_ARGSLOT + sizeof(Value)) { + return; + } + + LDefinition::Type type = virtualRegisters[vreg] + ? virtualRegisters[vreg]->type() + : LDefinition::GENERAL; + + switch (type) { + case LDefinition::OBJECT: + MOZ_ASSERT(safepoint->hasGcPointer(alloc)); + break; + case LDefinition::STACKRESULTS: + MOZ_ASSERT(safepoint->hasAllGcPointersFromStackArea(alloc)); + break; + case LDefinition::SLOTS: + MOZ_ASSERT(safepoint->hasSlotsOrElementsPointer(alloc)); + break; +# ifdef JS_NUNBOX32 + // Do not assert that safepoint information for nunbox types is complete, + // as if a vreg for a value's components are copied in multiple places + // then the safepoint information may not reflect all copies. All copies + // of payloads must be reflected, however, for generational GC. + case LDefinition::TYPE: + break; + case LDefinition::PAYLOAD: + MOZ_ASSERT(safepoint->hasNunboxPayload(alloc)); + break; +# else + case LDefinition::BOX: + MOZ_ASSERT(safepoint->hasBoxedValue(alloc)); + break; +# endif + default: + break; + } +} + +bool AllocationIntegrityState::addPredecessor(LBlock* block, uint32_t vreg, + LAllocation alloc) { + // There is no need to reanalyze if we have already seen this predecessor. + // We share the seen allocations across analysis of each use, as there will + // likely be common ground between different uses of the same vreg. + IntegrityItem item; + item.block = block; + item.vreg = vreg; + item.alloc = alloc; + item.index = seen.count(); + + IntegrityItemSet::AddPtr p = seen.lookupForAdd(item); + if (p) { + return true; + } + if (!seen.add(p, item)) { + return false; + } + + return worklist.append(item); +} + +void AllocationIntegrityState::dump() { +# ifdef JS_JITSPEW + JitSpewCont(JitSpew_RegAlloc, "\n"); + JitSpew(JitSpew_RegAlloc, "Register Allocation Integrity State:"); + + for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) { + LBlock* block = graph.getBlock(blockIndex); + MBasicBlock* mir = block->mir(); + + JitSpewHeader(JitSpew_RegAlloc); + JitSpewCont(JitSpew_RegAlloc, " Block %lu", + static_cast<unsigned long>(blockIndex)); + for (size_t i = 0; i < mir->numSuccessors(); i++) { + JitSpewCont(JitSpew_RegAlloc, " [successor %u]", + mir->getSuccessor(i)->id()); + } + JitSpewCont(JitSpew_RegAlloc, "\n"); + + for (size_t i = 0; i < block->numPhis(); i++) { + const InstructionInfo& info = blocks[blockIndex].phis[i]; + LPhi* phi = block->getPhi(i); + CodePosition input(block->getPhi(0)->id(), CodePosition::INPUT); + CodePosition output(block->getPhi(block->numPhis() - 1)->id(), + CodePosition::OUTPUT); + + JitSpewHeader(JitSpew_RegAlloc); + JitSpewCont(JitSpew_RegAlloc, " %u-%u Phi [def %s] ", input.bits(), + output.bits(), phi->getDef(0)->toString().get()); + for (size_t j = 0; j < phi->numOperands(); j++) { + JitSpewCont(JitSpew_RegAlloc, " [use %s]", + info.inputs[j].toString().get()); + } + JitSpewCont(JitSpew_RegAlloc, "\n"); + } + + for (LInstructionIterator iter = block->begin(); iter != block->end(); + iter++) { + LInstruction* ins = *iter; + const InstructionInfo& info = instructions[ins->id()]; + + CodePosition input(ins->id(), CodePosition::INPUT); + CodePosition output(ins->id(), CodePosition::OUTPUT); + + JitSpewHeader(JitSpew_RegAlloc); + JitSpewCont(JitSpew_RegAlloc, " "); + if (input != CodePosition::MIN) { + JitSpewCont(JitSpew_RegAlloc, "%u-%u ", input.bits(), output.bits()); + } + JitSpewCont(JitSpew_RegAlloc, "%s", ins->opName()); + + if (ins->isMoveGroup()) { + LMoveGroup* group = ins->toMoveGroup(); + for (int i = group->numMoves() - 1; i >= 0; i--) { + JitSpewCont(JitSpew_RegAlloc, " [%s <- %s]", + group->getMove(i).to().toString().get(), + group->getMove(i).from().toString().get()); + } + JitSpewCont(JitSpew_RegAlloc, "\n"); + continue; + } + + for (size_t i = 0; i < ins->numDefs(); i++) { + JitSpewCont(JitSpew_RegAlloc, " [def %s]", + ins->getDef(i)->toString().get()); + } + + for (size_t i = 0; i < ins->numTemps(); i++) { + LDefinition* temp = ins->getTemp(i); + if (!temp->isBogusTemp()) { + JitSpewCont(JitSpew_RegAlloc, " [temp v%u %s]", + info.temps[i].virtualRegister(), temp->toString().get()); + } + } + + size_t index = 0; + for (LInstruction::InputIterator alloc(*ins); alloc.more(); + alloc.next()) { + JitSpewCont(JitSpew_RegAlloc, " [use %s", + info.inputs[index++].toString().get()); + if (!alloc->isConstant()) { + JitSpewCont(JitSpew_RegAlloc, " %s", alloc->toString().get()); + } + JitSpewCont(JitSpew_RegAlloc, "]"); + } + + JitSpewCont(JitSpew_RegAlloc, "\n"); + } + } + + // Print discovered allocations at the ends of blocks, in the order they + // were discovered. + + Vector<IntegrityItem, 20, SystemAllocPolicy> seenOrdered; + if (!seenOrdered.appendN(IntegrityItem(), seen.count())) { + fprintf(stderr, "OOM while dumping allocations\n"); + return; + } + + for (IntegrityItemSet::Enum iter(seen); !iter.empty(); iter.popFront()) { + IntegrityItem item = iter.front(); + seenOrdered[item.index] = item; + } + + if (!seenOrdered.empty()) { + fprintf(stderr, "Intermediate Allocations:\n"); + + for (size_t i = 0; i < seenOrdered.length(); i++) { + IntegrityItem item = seenOrdered[i]; + fprintf(stderr, " block %u reg v%u alloc %s\n", item.block->mir()->id(), + item.vreg, item.alloc.toString().get()); + } + } + + fprintf(stderr, "\n"); +# endif +} +#endif // DEBUG + +const CodePosition CodePosition::MAX(UINT_MAX); +const CodePosition CodePosition::MIN(0); + +bool RegisterAllocator::init() { + if (!insData.init(mir, graph.numInstructions())) { + return false; + } + + if (!entryPositions.reserve(graph.numBlocks()) || + !exitPositions.reserve(graph.numBlocks())) { + return false; + } + + for (size_t i = 0; i < graph.numBlocks(); i++) { + LBlock* block = graph.getBlock(i); + for (LInstructionIterator ins = block->begin(); ins != block->end(); + ins++) { + insData[ins->id()] = *ins; + } + for (size_t j = 0; j < block->numPhis(); j++) { + LPhi* phi = block->getPhi(j); + insData[phi->id()] = phi; + } + + CodePosition entry = + block->numPhis() != 0 + ? CodePosition(block->getPhi(0)->id(), CodePosition::INPUT) + : inputOf(block->firstInstructionWithId()); + CodePosition exit = outputOf(block->lastInstructionWithId()); + + MOZ_ASSERT(block->mir()->id() == i); + entryPositions.infallibleAppend(entry); + exitPositions.infallibleAppend(exit); + } + + return true; +} + +LMoveGroup* RegisterAllocator::getInputMoveGroup(LInstruction* ins) { + MOZ_ASSERT(!ins->fixReuseMoves()); + if (ins->inputMoves()) { + return ins->inputMoves(); + } + + LMoveGroup* moves = LMoveGroup::New(alloc()); + ins->setInputMoves(moves); + ins->block()->insertBefore(ins, moves); + return moves; +} + +LMoveGroup* RegisterAllocator::getFixReuseMoveGroup(LInstruction* ins) { + if (ins->fixReuseMoves()) { + return ins->fixReuseMoves(); + } + + LMoveGroup* moves = LMoveGroup::New(alloc()); + ins->setFixReuseMoves(moves); + ins->block()->insertBefore(ins, moves); + return moves; +} + +LMoveGroup* RegisterAllocator::getMoveGroupAfter(LInstruction* ins) { + if (ins->movesAfter()) { + return ins->movesAfter(); + } + + LMoveGroup* moves = LMoveGroup::New(alloc()); + ins->setMovesAfter(moves); + + ins->block()->insertAfter(ins, moves); + return moves; +} + +void RegisterAllocator::dumpInstructions(const char* who) { +#ifdef JS_JITSPEW + JitSpew(JitSpew_RegAlloc, "LIR instructions %s", who); + + for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) { + LBlock* block = graph.getBlock(blockIndex); + MBasicBlock* mir = block->mir(); + + JitSpewHeader(JitSpew_RegAlloc); + JitSpewCont(JitSpew_RegAlloc, " Block %lu", + static_cast<unsigned long>(blockIndex)); + for (size_t i = 0; i < mir->numSuccessors(); i++) { + JitSpewCont(JitSpew_RegAlloc, " [successor %u]", + mir->getSuccessor(i)->id()); + } + JitSpewCont(JitSpew_RegAlloc, "\n"); + + for (size_t i = 0; i < block->numPhis(); i++) { + LPhi* phi = block->getPhi(i); + + JitSpewHeader(JitSpew_RegAlloc); + JitSpewCont(JitSpew_RegAlloc, " %u-%u Phi [def %s]", + inputOf(phi).bits(), outputOf(phi).bits(), + phi->getDef(0)->toString().get()); + for (size_t j = 0; j < phi->numOperands(); j++) { + JitSpewCont(JitSpew_RegAlloc, " [use %s]", + phi->getOperand(j)->toString().get()); + } + JitSpewCont(JitSpew_RegAlloc, "\n"); + } + + for (LInstructionIterator iter = block->begin(); iter != block->end(); + iter++) { + LInstruction* ins = *iter; + + JitSpewHeader(JitSpew_RegAlloc); + JitSpewCont(JitSpew_RegAlloc, " "); + if (ins->id() != 0) { + JitSpewCont(JitSpew_RegAlloc, "%u-%u ", inputOf(ins).bits(), + outputOf(ins).bits()); + } + JitSpewCont(JitSpew_RegAlloc, "%s", ins->opName()); + + if (ins->isMoveGroup()) { + LMoveGroup* group = ins->toMoveGroup(); + for (int i = group->numMoves() - 1; i >= 0; i--) { + // Use two printfs, as LAllocation::toString is not reentant. + JitSpewCont(JitSpew_RegAlloc, " [%s", + group->getMove(i).to().toString().get()); + JitSpewCont(JitSpew_RegAlloc, " <- %s]", + group->getMove(i).from().toString().get()); + } + JitSpewCont(JitSpew_RegAlloc, "\n"); + continue; + } + + for (size_t i = 0; i < ins->numDefs(); i++) { + JitSpewCont(JitSpew_RegAlloc, " [def %s]", + ins->getDef(i)->toString().get()); + } + + for (size_t i = 0; i < ins->numTemps(); i++) { + LDefinition* temp = ins->getTemp(i); + if (!temp->isBogusTemp()) { + JitSpewCont(JitSpew_RegAlloc, " [temp %s]", temp->toString().get()); + } + } + + for (LInstruction::InputIterator alloc(*ins); alloc.more(); + alloc.next()) { + if (!alloc->isBogus()) { + JitSpewCont(JitSpew_RegAlloc, " [use %s]", alloc->toString().get()); + } + } + + JitSpewCont(JitSpew_RegAlloc, "\n"); + } + } + JitSpewCont(JitSpew_RegAlloc, "\n"); +#endif // JS_JITSPEW +} |