diff options
Diffstat (limited to 'js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp')
| -rw-r--r-- | js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp | 528 |
1 files changed, 528 insertions, 0 deletions
diff --git a/js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp b/js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp new file mode 100644 index 0000000000..590bd90a37 --- /dev/null +++ b/js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp @@ -0,0 +1,528 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/x86-shared/MoveEmitter-x86-shared.h" + +#include "jit/MacroAssembler-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::Maybe; + +MoveEmitterX86::MoveEmitterX86(MacroAssembler& masm) + : inCycle_(false), masm(masm), pushedAtCycle_(-1) { + pushedAtStart_ = masm.framePushed(); +} + +// Examine the cycle in moves starting at position i. Determine if it's a +// simple cycle consisting of all register-to-register moves in a single class, +// and whether it can be implemented entirely by swaps. +size_t MoveEmitterX86::characterizeCycle(const MoveResolver& moves, size_t i, + bool* allGeneralRegs, + bool* allFloatRegs) { + size_t swapCount = 0; + + for (size_t j = i;; j++) { + const MoveOp& move = moves.getMove(j); + + // If it isn't a cycle of registers of the same kind, we won't be able + // to optimize it. + if (!move.to().isGeneralReg()) { + *allGeneralRegs = false; + } + if (!move.to().isFloatReg()) { + *allFloatRegs = false; + } + if (!*allGeneralRegs && !*allFloatRegs) { + return -1; + } + + // Stop iterating when we see the last one. + if (j != i && move.isCycleEnd()) { + break; + } + + // Check that this move is actually part of the cycle. This is + // over-conservative when there are multiple reads from the same source, + // but that's expected to be rare. + if (move.from() != moves.getMove(j + 1).to()) { + *allGeneralRegs = false; + *allFloatRegs = false; + return -1; + } + + swapCount++; + } + + // Check that the last move cycles back to the first move. + const MoveOp& move = moves.getMove(i + swapCount); + if (move.from() != moves.getMove(i).to()) { + *allGeneralRegs = false; + *allFloatRegs = false; + return -1; + } + + return swapCount; +} + +// If we can emit optimized code for the cycle in moves starting at position i, +// do so, and return true. +bool MoveEmitterX86::maybeEmitOptimizedCycle(const MoveResolver& moves, + size_t i, bool allGeneralRegs, + bool allFloatRegs, + size_t swapCount) { + if (allGeneralRegs && swapCount <= 2) { + // Use x86's swap-integer-registers instruction if we only have a few + // swaps. (x86 also has a swap between registers and memory but it's + // slow.) + for (size_t k = 0; k < swapCount; k++) { + masm.xchg(moves.getMove(i + k).to().reg(), + moves.getMove(i + k + 1).to().reg()); + } + return true; + } + + if (allFloatRegs && swapCount == 1) { + // There's no xchg for xmm registers, but if we only need a single swap, + // it's cheap to do an XOR swap. + FloatRegister a = moves.getMove(i).to().floatReg(); + FloatRegister b = moves.getMove(i + 1).to().floatReg(); + masm.vxorpd(a, b, b); + masm.vxorpd(b, a, a); + masm.vxorpd(a, b, b); + return true; + } + + return false; +} + +void MoveEmitterX86::emit(const MoveResolver& moves) { +#if defined(JS_CODEGEN_X86) && defined(DEBUG) + // Clobber any scratch register we have, to make regalloc bugs more visible. + if (scratchRegister_.isSome()) { + masm.mov(ImmWord(0xdeadbeef), scratchRegister_.value()); + } +#endif + + for (size_t i = 0; i < moves.numMoves(); i++) { +#if defined(JS_CODEGEN_X86) && defined(DEBUG) + if (!scratchRegister_.isSome()) { + Maybe<Register> reg = findScratchRegister(moves, i); + if (reg.isSome()) { + masm.mov(ImmWord(0xdeadbeef), reg.value()); + } + } +#endif + + const MoveOp& move = moves.getMove(i); + const MoveOperand& from = move.from(); + const MoveOperand& to = move.to(); + + if (move.isCycleEnd()) { + MOZ_ASSERT(inCycle_); + completeCycle(to, move.type()); + inCycle_ = false; + continue; + } + + if (move.isCycleBegin()) { + MOZ_ASSERT(!inCycle_); + + // Characterize the cycle. + bool allGeneralRegs = true, allFloatRegs = true; + size_t swapCount = + characterizeCycle(moves, i, &allGeneralRegs, &allFloatRegs); + + // Attempt to optimize it to avoid using the stack. + if (maybeEmitOptimizedCycle(moves, i, allGeneralRegs, allFloatRegs, + swapCount)) { + i += swapCount; + continue; + } + + // Otherwise use the stack. + breakCycle(to, move.endCycleType()); + inCycle_ = true; + } + + // A normal move which is not part of a cycle. + switch (move.type()) { + case MoveOp::FLOAT32: + emitFloat32Move(from, to); + break; + case MoveOp::DOUBLE: + emitDoubleMove(from, to); + break; + case MoveOp::INT32: + emitInt32Move(from, to, moves, i); + break; + case MoveOp::GENERAL: + emitGeneralMove(from, to, moves, i); + break; + case MoveOp::SIMD128: + emitSimd128Move(from, to); + break; + default: + MOZ_CRASH("Unexpected move type"); + } + } +} + +MoveEmitterX86::~MoveEmitterX86() { assertDone(); } + +Address MoveEmitterX86::cycleSlot() { + if (pushedAtCycle_ == -1) { + // Reserve stack for cycle resolution + static_assert(SpillSlotSize == 16); + masm.reserveStack(SpillSlotSize); + pushedAtCycle_ = masm.framePushed(); + } + + return Address(StackPointer, masm.framePushed() - pushedAtCycle_); +} + +Address MoveEmitterX86::toAddress(const MoveOperand& operand) const { + if (operand.base() != StackPointer) { + return Address(operand.base(), operand.disp()); + } + + MOZ_ASSERT(operand.disp() >= 0); + + // Otherwise, the stack offset may need to be adjusted. + return Address(StackPointer, + operand.disp() + (masm.framePushed() - pushedAtStart_)); +} + +// Warning, do not use the resulting operand with pop instructions, since they +// compute the effective destination address after altering the stack pointer. +// Use toPopOperand if an Operand is needed for a pop. +Operand MoveEmitterX86::toOperand(const MoveOperand& operand) const { + if (operand.isMemoryOrEffectiveAddress()) { + return Operand(toAddress(operand)); + } + if (operand.isGeneralReg()) { + return Operand(operand.reg()); + } + + MOZ_ASSERT(operand.isFloatReg()); + return Operand(operand.floatReg()); +} + +// This is the same as toOperand except that it computes an Operand suitable for +// use in a pop. +Operand MoveEmitterX86::toPopOperand(const MoveOperand& operand) const { + if (operand.isMemory()) { + if (operand.base() != StackPointer) { + return Operand(operand.base(), operand.disp()); + } + + MOZ_ASSERT(operand.disp() >= 0); + + // Otherwise, the stack offset may need to be adjusted. + // Note the adjustment by the stack slot here, to offset for the fact that + // pop computes its effective address after incrementing the stack pointer. + return Operand( + StackPointer, + operand.disp() + (masm.framePushed() - sizeof(void*) - pushedAtStart_)); + } + if (operand.isGeneralReg()) { + return Operand(operand.reg()); + } + + MOZ_ASSERT(operand.isFloatReg()); + return Operand(operand.floatReg()); +} + +void MoveEmitterX86::breakCycle(const MoveOperand& to, MoveOp::Type type) { + // There is some pattern: + // (A -> B) + // (B -> A) + // + // This case handles (A -> B), which we reach first. We save B, then allow + // the original move to continue. + switch (type) { + case MoveOp::SIMD128: + if (to.isMemory()) { + ScratchSimd128Scope scratch(masm); + masm.loadUnalignedSimd128(toAddress(to), scratch); + masm.storeUnalignedSimd128(scratch, cycleSlot()); + } else { + masm.storeUnalignedSimd128(to.floatReg(), cycleSlot()); + } + break; + case MoveOp::FLOAT32: + if (to.isMemory()) { + ScratchFloat32Scope scratch(masm); + masm.loadFloat32(toAddress(to), scratch); + masm.storeFloat32(scratch, cycleSlot()); + } else { + masm.storeFloat32(to.floatReg(), cycleSlot()); + } + break; + case MoveOp::DOUBLE: + if (to.isMemory()) { + ScratchDoubleScope scratch(masm); + masm.loadDouble(toAddress(to), scratch); + masm.storeDouble(scratch, cycleSlot()); + } else { + masm.storeDouble(to.floatReg(), cycleSlot()); + } + break; + case MoveOp::INT32: +#ifdef JS_CODEGEN_X64 + // x64 can't pop to a 32-bit destination, so don't push. + if (to.isMemory()) { + masm.load32(toAddress(to), ScratchReg); + masm.store32(ScratchReg, cycleSlot()); + } else { + masm.store32(to.reg(), cycleSlot()); + } + break; +#endif + case MoveOp::GENERAL: + masm.Push(toOperand(to)); + break; + default: + MOZ_CRASH("Unexpected move type"); + } +} + +void MoveEmitterX86::completeCycle(const MoveOperand& to, MoveOp::Type type) { + // There is some pattern: + // (A -> B) + // (B -> A) + // + // This case handles (B -> A), which we reach last. We emit a move from the + // saved value of B, to A. + switch (type) { + case MoveOp::SIMD128: + MOZ_ASSERT(pushedAtCycle_ != -1); + MOZ_ASSERT(pushedAtCycle_ - pushedAtStart_ >= Simd128DataSize); + if (to.isMemory()) { + ScratchSimd128Scope scratch(masm); + masm.loadUnalignedSimd128(cycleSlot(), scratch); + masm.storeUnalignedSimd128(scratch, toAddress(to)); + } else { + masm.loadUnalignedSimd128(cycleSlot(), to.floatReg()); + } + break; + case MoveOp::FLOAT32: + MOZ_ASSERT(pushedAtCycle_ != -1); + MOZ_ASSERT(pushedAtCycle_ - pushedAtStart_ >= sizeof(float)); + if (to.isMemory()) { + ScratchFloat32Scope scratch(masm); + masm.loadFloat32(cycleSlot(), scratch); + masm.storeFloat32(scratch, toAddress(to)); + } else { + masm.loadFloat32(cycleSlot(), to.floatReg()); + } + break; + case MoveOp::DOUBLE: + MOZ_ASSERT(pushedAtCycle_ != -1); + MOZ_ASSERT(pushedAtCycle_ - pushedAtStart_ >= sizeof(double)); + if (to.isMemory()) { + ScratchDoubleScope scratch(masm); + masm.loadDouble(cycleSlot(), scratch); + masm.storeDouble(scratch, toAddress(to)); + } else { + masm.loadDouble(cycleSlot(), to.floatReg()); + } + break; + case MoveOp::INT32: +#ifdef JS_CODEGEN_X64 + MOZ_ASSERT(pushedAtCycle_ != -1); + MOZ_ASSERT(pushedAtCycle_ - pushedAtStart_ >= sizeof(int32_t)); + // x64 can't pop to a 32-bit destination. + if (to.isMemory()) { + masm.load32(cycleSlot(), ScratchReg); + masm.store32(ScratchReg, toAddress(to)); + } else { + masm.load32(cycleSlot(), to.reg()); + } + break; +#endif + case MoveOp::GENERAL: + MOZ_ASSERT(masm.framePushed() - pushedAtStart_ >= sizeof(intptr_t)); + masm.Pop(toPopOperand(to)); + break; + default: + MOZ_CRASH("Unexpected move type"); + } +} + +void MoveEmitterX86::emitInt32Move(const MoveOperand& from, + const MoveOperand& to, + const MoveResolver& moves, size_t i) { + if (from.isGeneralReg()) { + masm.move32(from.reg(), toOperand(to)); + } else if (to.isGeneralReg()) { + MOZ_ASSERT(from.isMemory()); + masm.load32(toAddress(from), to.reg()); + } else { + // Memory to memory gpr move. + MOZ_ASSERT(from.isMemory()); + Maybe<Register> reg = findScratchRegister(moves, i); + if (reg.isSome()) { + masm.load32(toAddress(from), reg.value()); + masm.move32(reg.value(), toOperand(to)); + } else { + // No scratch register available; bounce it off the stack. + masm.Push(toOperand(from)); + masm.Pop(toPopOperand(to)); + } + } +} + +void MoveEmitterX86::emitGeneralMove(const MoveOperand& from, + const MoveOperand& to, + const MoveResolver& moves, size_t i) { + if (from.isGeneralReg()) { + masm.mov(from.reg(), toOperand(to)); + } else if (to.isGeneralReg()) { + MOZ_ASSERT(from.isMemoryOrEffectiveAddress()); + if (from.isMemory()) { + masm.loadPtr(toAddress(from), to.reg()); + } else { + masm.lea(toOperand(from), to.reg()); + } + } else if (from.isMemory()) { + // Memory to memory gpr move. + Maybe<Register> reg = findScratchRegister(moves, i); + if (reg.isSome()) { + masm.loadPtr(toAddress(from), reg.value()); + masm.mov(reg.value(), toOperand(to)); + } else { + // No scratch register available; bounce it off the stack. + masm.Push(toOperand(from)); + masm.Pop(toPopOperand(to)); + } + } else { + // Effective address to memory move. + MOZ_ASSERT(from.isEffectiveAddress()); + Maybe<Register> reg = findScratchRegister(moves, i); + if (reg.isSome()) { + masm.lea(toOperand(from), reg.value()); + masm.mov(reg.value(), toOperand(to)); + } else { + // This is tricky without a scratch reg. We can't do an lea. Bounce the + // base register off the stack, then add the offset in place. Note that + // this clobbers FLAGS! + masm.Push(from.base()); + masm.Pop(toPopOperand(to)); + MOZ_ASSERT(to.isMemoryOrEffectiveAddress()); + masm.addPtr(Imm32(from.disp()), toAddress(to)); + } + } +} + +void MoveEmitterX86::emitFloat32Move(const MoveOperand& from, + const MoveOperand& to) { + MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg().isSingle()); + MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg().isSingle()); + + if (from.isFloatReg()) { + if (to.isFloatReg()) { + masm.moveFloat32(from.floatReg(), to.floatReg()); + } else { + masm.storeFloat32(from.floatReg(), toAddress(to)); + } + } else if (to.isFloatReg()) { + masm.loadFloat32(toAddress(from), to.floatReg()); + } else { + // Memory to memory move. + MOZ_ASSERT(from.isMemory()); + ScratchFloat32Scope scratch(masm); + masm.loadFloat32(toAddress(from), scratch); + masm.storeFloat32(scratch, toAddress(to)); + } +} + +void MoveEmitterX86::emitDoubleMove(const MoveOperand& from, + const MoveOperand& to) { + MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg().isDouble()); + MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg().isDouble()); + + if (from.isFloatReg()) { + if (to.isFloatReg()) { + masm.moveDouble(from.floatReg(), to.floatReg()); + } else { + masm.storeDouble(from.floatReg(), toAddress(to)); + } + } else if (to.isFloatReg()) { + masm.loadDouble(toAddress(from), to.floatReg()); + } else { + // Memory to memory move. + MOZ_ASSERT(from.isMemory()); + ScratchDoubleScope scratch(masm); + masm.loadDouble(toAddress(from), scratch); + masm.storeDouble(scratch, toAddress(to)); + } +} + +void MoveEmitterX86::emitSimd128Move(const MoveOperand& from, + const MoveOperand& to) { + MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg().isSimd128()); + MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg().isSimd128()); + + if (from.isFloatReg()) { + if (to.isFloatReg()) { + masm.moveSimd128(from.floatReg(), to.floatReg()); + } else { + masm.storeUnalignedSimd128(from.floatReg(), toAddress(to)); + } + } else if (to.isFloatReg()) { + masm.loadUnalignedSimd128(toAddress(from), to.floatReg()); + } else { + // Memory to memory move. + MOZ_ASSERT(from.isMemory()); + ScratchSimd128Scope scratch(masm); + masm.loadUnalignedSimd128(toAddress(from), scratch); + masm.storeUnalignedSimd128(scratch, toAddress(to)); + } +} + +void MoveEmitterX86::assertDone() { MOZ_ASSERT(!inCycle_); } + +void MoveEmitterX86::finish() { + assertDone(); + + masm.freeStack(masm.framePushed() - pushedAtStart_); +} + +Maybe<Register> MoveEmitterX86::findScratchRegister(const MoveResolver& moves, + size_t initial) { +#ifdef JS_CODEGEN_X86 + if (scratchRegister_.isSome()) { + return scratchRegister_; + } + + // All registers are either in use by this move group or are live + // afterwards. Look through the remaining moves for a register which is + // clobbered before it is used, and is thus dead at this point. + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + for (size_t i = initial; i < moves.numMoves(); i++) { + const MoveOp& move = moves.getMove(i); + if (move.from().isGeneralReg()) { + regs.takeUnchecked(move.from().reg()); + } else if (move.from().isMemoryOrEffectiveAddress()) { + regs.takeUnchecked(move.from().base()); + } + if (move.to().isGeneralReg()) { + if (i != initial && !move.isCycleBegin() && regs.has(move.to().reg())) { + return mozilla::Some(move.to().reg()); + } + regs.takeUnchecked(move.to().reg()); + } else if (move.to().isMemoryOrEffectiveAddress()) { + regs.takeUnchecked(move.to().base()); + } + } + + return mozilla::Nothing(); +#else + return mozilla::Some(ScratchReg); +#endif +} |