From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- js/src/jit/ABIArgGenerator.h | 109 + js/src/jit/ABIFunctionList-inl.h | 252 + js/src/jit/ABIFunctions.h | 73 + js/src/jit/AliasAnalysis.cpp | 317 + js/src/jit/AliasAnalysis.h | 64 + js/src/jit/AlignmentMaskAnalysis.cpp | 97 + js/src/jit/AlignmentMaskAnalysis.h | 27 + js/src/jit/Assembler.h | 34 + js/src/jit/AtomicOp.h | 98 + js/src/jit/AtomicOperations.h | 352 + js/src/jit/AutoWritableJitCode.h | 88 + js/src/jit/BacktrackingAllocator.cpp | 4676 +++++ js/src/jit/BacktrackingAllocator.h | 844 + js/src/jit/Bailouts.cpp | 352 + js/src/jit/Bailouts.h | 227 + js/src/jit/BaselineBailouts.cpp | 2092 +++ js/src/jit/BaselineCacheIRCompiler.cpp | 4083 ++++ js/src/jit/BaselineCacheIRCompiler.h | 150 + js/src/jit/BaselineCodeGen.cpp | 6897 +++++++ js/src/jit/BaselineCodeGen.h | 521 + js/src/jit/BaselineDebugModeOSR.cpp | 561 + js/src/jit/BaselineDebugModeOSR.h | 30 + js/src/jit/BaselineFrame-inl.h | 131 + js/src/jit/BaselineFrame.cpp | 180 + js/src/jit/BaselineFrame.h | 373 + js/src/jit/BaselineFrameInfo-inl.h | 50 + js/src/jit/BaselineFrameInfo.cpp | 239 + js/src/jit/BaselineFrameInfo.h | 435 + js/src/jit/BaselineIC.cpp | 2497 +++ js/src/jit/BaselineIC.h | 439 + js/src/jit/BaselineICList.h | 49 + js/src/jit/BaselineJIT.cpp | 1008 + js/src/jit/BaselineJIT.h | 593 + js/src/jit/BitSet.cpp | 111 + js/src/jit/BitSet.h | 167 + js/src/jit/BytecodeAnalysis.cpp | 257 + js/src/jit/BytecodeAnalysis.h | 83 + js/src/jit/CacheIR.cpp | 13193 +++++++++++++ js/src/jit/CacheIR.h | 528 + js/src/jit/CacheIRCloner.h | 82 + js/src/jit/CacheIRCompiler.cpp | 9638 ++++++++++ js/src/jit/CacheIRCompiler.h | 1314 ++ js/src/jit/CacheIRGenerator.h | 912 + js/src/jit/CacheIRHealth.cpp | 416 + js/src/jit/CacheIRHealth.h | 109 + js/src/jit/CacheIROps.yaml | 3086 +++ js/src/jit/CacheIRReader.h | 155 + js/src/jit/CacheIRSpewer.cpp | 441 + js/src/jit/CacheIRSpewer.h | 116 + js/src/jit/CacheIRWriter.h | 642 + js/src/jit/CalleeToken.h | 66 + js/src/jit/CodeGenerator.cpp | 18631 +++++++++++++++++++ js/src/jit/CodeGenerator.h | 447 + js/src/jit/CompactBuffer.h | 254 + js/src/jit/CompileInfo.h | 382 + js/src/jit/CompileWrappers.cpp | 219 + js/src/jit/CompileWrappers.h | 176 + js/src/jit/Disassemble.cpp | 109 + js/src/jit/Disassemble.h | 24 + js/src/jit/EdgeCaseAnalysis.cpp | 50 + js/src/jit/EdgeCaseAnalysis.h | 28 + js/src/jit/EffectiveAddressAnalysis.cpp | 256 + js/src/jit/EffectiveAddressAnalysis.h | 33 + js/src/jit/ExecutableAllocator.cpp | 329 + js/src/jit/ExecutableAllocator.h | 205 + js/src/jit/FixedList.h | 99 + js/src/jit/FlushICache.cpp | 132 + js/src/jit/FlushICache.h | 92 + js/src/jit/FoldLinearArithConstants.cpp | 112 + js/src/jit/FoldLinearArithConstants.h | 21 + js/src/jit/GenerateAtomicOperations.py | 873 + js/src/jit/GenerateCacheIRFiles.py | 539 + js/src/jit/GenerateLIRFiles.py | 298 + js/src/jit/GenerateMIRFiles.py | 404 + js/src/jit/ICState.h | 216 + js/src/jit/ICStubSpace.h | 66 + js/src/jit/InlinableNatives.cpp | 300 + js/src/jit/InlinableNatives.h | 240 + js/src/jit/InlineList.h | 590 + js/src/jit/InlineScriptTree-inl.h | 67 + js/src/jit/InlineScriptTree.h | 112 + js/src/jit/InstructionReordering.cpp | 248 + js/src/jit/InstructionReordering.h | 20 + js/src/jit/InterpreterEntryTrampoline.cpp | 269 + js/src/jit/InterpreterEntryTrampoline.h | 79 + js/src/jit/Invalidation.h | 59 + js/src/jit/Ion.cpp | 2631 +++ js/src/jit/Ion.h | 154 + js/src/jit/IonAnalysis.cpp | 4934 +++++ js/src/jit/IonAnalysis.h | 193 + js/src/jit/IonCacheIRCompiler.cpp | 2140 +++ js/src/jit/IonCacheIRCompiler.h | 93 + js/src/jit/IonCompileTask.cpp | 203 + js/src/jit/IonCompileTask.h | 89 + js/src/jit/IonIC.cpp | 727 + js/src/jit/IonIC.h | 664 + js/src/jit/IonOptimizationLevels.cpp | 141 + js/src/jit/IonOptimizationLevels.h | 203 + js/src/jit/IonScript.h | 590 + js/src/jit/IonTypes.h | 1108 ++ js/src/jit/JSJitFrameIter-inl.h | 65 + js/src/jit/JSJitFrameIter.cpp | 798 + js/src/jit/JSJitFrameIter.h | 802 + js/src/jit/JSONSpewer.cpp | 287 + js/src/jit/JSONSpewer.h | 48 + js/src/jit/Jit.cpp | 214 + js/src/jit/Jit.h | 41 + js/src/jit/JitAllocPolicy.h | 179 + js/src/jit/JitCode.h | 171 + js/src/jit/JitCommon.h | 53 + js/src/jit/JitContext.cpp | 161 + js/src/jit/JitContext.h | 168 + js/src/jit/JitFrames-inl.h | 32 + js/src/jit/JitFrames.cpp | 2570 +++ js/src/jit/JitFrames.h | 748 + js/src/jit/JitHints-inl.h | 60 + js/src/jit/JitHints.h | 56 + js/src/jit/JitOptions.cpp | 414 + js/src/jit/JitOptions.h | 184 + js/src/jit/JitRealm.h | 189 + js/src/jit/JitRuntime.h | 451 + js/src/jit/JitScript-inl.h | 43 + js/src/jit/JitScript.cpp | 732 + js/src/jit/JitScript.h | 543 + js/src/jit/JitSpewer.cpp | 660 + js/src/jit/JitSpewer.h | 286 + js/src/jit/JitZone.h | 208 + js/src/jit/JitcodeMap.cpp | 1145 ++ js/src/jit/JitcodeMap.h | 808 + js/src/jit/Jitdump.h | 78 + js/src/jit/KnownClass.cpp | 109 + js/src/jit/KnownClass.h | 36 + js/src/jit/LICM.cpp | 367 + js/src/jit/LICM.h | 23 + js/src/jit/LIR.cpp | 780 + js/src/jit/LIR.h | 2000 ++ js/src/jit/LIROps.yaml | 3972 ++++ js/src/jit/Label.cpp | 29 + js/src/jit/Label.h | 106 + js/src/jit/Linker.cpp | 79 + js/src/jit/Linker.h | 52 + js/src/jit/Lowering.cpp | 7172 +++++++ js/src/jit/Lowering.h | 91 + js/src/jit/MIR.cpp | 7261 ++++++++ js/src/jit/MIR.h | 11613 ++++++++++++ js/src/jit/MIRGenerator.h | 183 + js/src/jit/MIRGraph.cpp | 1414 ++ js/src/jit/MIRGraph.h | 901 + js/src/jit/MIROps.yaml | 3064 +++ js/src/jit/MachineState.h | 110 + js/src/jit/MacroAssembler-inl.h | 1090 ++ js/src/jit/MacroAssembler.cpp | 6671 +++++++ js/src/jit/MacroAssembler.h | 5611 ++++++ js/src/jit/MoveEmitter.h | 32 + js/src/jit/MoveResolver.cpp | 443 + js/src/jit/MoveResolver.h | 309 + js/src/jit/PcScriptCache.h | 88 + js/src/jit/PerfSpewer.cpp | 1218 ++ js/src/jit/PerfSpewer.h | 207 + js/src/jit/ProcessExecutableMemory.cpp | 935 + js/src/jit/ProcessExecutableMemory.h | 109 + js/src/jit/RangeAnalysis.cpp | 3679 ++++ js/src/jit/RangeAnalysis.h | 683 + js/src/jit/ReciprocalMulConstants.cpp | 94 + js/src/jit/ReciprocalMulConstants.h | 33 + js/src/jit/Recover.cpp | 2116 +++ js/src/jit/Recover.h | 964 + js/src/jit/RegExpStubConstants.h | 36 + js/src/jit/RegisterAllocator.cpp | 669 + js/src/jit/RegisterAllocator.h | 314 + js/src/jit/RegisterSets.h | 1332 ++ js/src/jit/Registers.h | 299 + js/src/jit/RematerializedFrame-inl.h | 23 + js/src/jit/RematerializedFrame.cpp | 221 + js/src/jit/RematerializedFrame.h | 222 + js/src/jit/SafepointIndex-inl.h | 22 + js/src/jit/SafepointIndex.cpp | 20 + js/src/jit/SafepointIndex.h | 76 + js/src/jit/Safepoints.cpp | 559 + js/src/jit/Safepoints.h | 129 + js/src/jit/ScalarReplacement.cpp | 3086 +++ js/src/jit/ScalarReplacement.h | 22 + js/src/jit/ScalarTypeUtils.h | 41 + js/src/jit/ScriptFromCalleeToken.h | 33 + js/src/jit/SharedICHelpers-inl.h | 36 + js/src/jit/SharedICHelpers.h | 36 + js/src/jit/SharedICRegisters.h | 38 + js/src/jit/ShuffleAnalysis.cpp | 747 + js/src/jit/ShuffleAnalysis.h | 147 + js/src/jit/Simulator.h | 32 + js/src/jit/Sink.cpp | 255 + js/src/jit/Sink.h | 22 + js/src/jit/Snapshots.cpp | 605 + js/src/jit/Snapshots.h | 529 + js/src/jit/StackSlotAllocator.h | 133 + js/src/jit/TemplateObject-inl.h | 126 + js/src/jit/TemplateObject.h | 77 + js/src/jit/Trampoline.cpp | 260 + js/src/jit/TrialInlining.cpp | 928 + js/src/jit/TrialInlining.h | 194 + js/src/jit/TypeData.h | 54 + js/src/jit/TypePolicy.cpp | 1152 ++ js/src/jit/TypePolicy.h | 557 + js/src/jit/VMFunctionList-inl.h | 379 + js/src/jit/VMFunctions.cpp | 2940 +++ js/src/jit/VMFunctions.h | 713 + js/src/jit/ValueNumbering.cpp | 1338 ++ js/src/jit/ValueNumbering.h | 123 + js/src/jit/WarpBuilder.cpp | 3576 ++++ js/src/jit/WarpBuilder.h | 326 + js/src/jit/WarpBuilderShared.cpp | 99 + js/src/jit/WarpBuilderShared.h | 425 + js/src/jit/WarpCacheIRTranspiler.cpp | 5809 ++++++ js/src/jit/WarpCacheIRTranspiler.h | 33 + js/src/jit/WarpOracle.cpp | 1226 ++ js/src/jit/WarpOracle.h | 68 + js/src/jit/WarpSnapshot.cpp | 408 + js/src/jit/WarpSnapshot.h | 627 + js/src/jit/WasmBCE.cpp | 139 + js/src/jit/WasmBCE.h | 33 + js/src/jit/XrayJitInfo.cpp | 17 + js/src/jit/arm/Architecture-arm.cpp | 540 + js/src/jit/arm/Architecture-arm.h | 733 + js/src/jit/arm/Assembler-arm.cpp | 2832 +++ js/src/jit/arm/Assembler-arm.h | 2296 +++ js/src/jit/arm/CodeGenerator-arm.cpp | 3154 ++++ js/src/jit/arm/CodeGenerator-arm.h | 172 + js/src/jit/arm/DoubleEntryTable.tbl | 257 + js/src/jit/arm/LIR-arm.h | 511 + js/src/jit/arm/Lowering-arm.cpp | 1223 ++ js/src/jit/arm/Lowering-arm.h | 118 + js/src/jit/arm/MacroAssembler-arm-inl.h | 2582 +++ js/src/jit/arm/MacroAssembler-arm.cpp | 6382 +++++++ js/src/jit/arm/MacroAssembler-arm.h | 1392 ++ js/src/jit/arm/MoveEmitter-arm.cpp | 413 + js/src/jit/arm/MoveEmitter-arm.h | 70 + js/src/jit/arm/SharedICHelpers-arm-inl.h | 79 + js/src/jit/arm/SharedICHelpers-arm.h | 80 + js/src/jit/arm/SharedICRegisters-arm.h | 52 + js/src/jit/arm/Simulator-arm.cpp | 5472 ++++++ js/src/jit/arm/Simulator-arm.h | 632 + js/src/jit/arm/Trampoline-arm.cpp | 831 + js/src/jit/arm/disasm/Constants-arm.cpp | 117 + js/src/jit/arm/disasm/Constants-arm.h | 684 + js/src/jit/arm/disasm/Disasm-arm.cpp | 2031 ++ js/src/jit/arm/disasm/Disasm-arm.h | 141 + js/src/jit/arm/gen-double-encoder-table.py | 35 + .../jit/arm/llvm-compiler-rt/arm/aeabi_idivmod.S | 27 + .../jit/arm/llvm-compiler-rt/arm/aeabi_uidivmod.S | 28 + js/src/jit/arm/llvm-compiler-rt/assembly.h | 67 + js/src/jit/arm64/Architecture-arm64.cpp | 129 + js/src/jit/arm64/Architecture-arm64.h | 773 + js/src/jit/arm64/Assembler-arm64.cpp | 609 + js/src/jit/arm64/Assembler-arm64.h | 793 + js/src/jit/arm64/CodeGenerator-arm64.cpp | 4245 +++++ js/src/jit/arm64/CodeGenerator-arm64.h | 135 + js/src/jit/arm64/LIR-arm64.h | 373 + js/src/jit/arm64/Lowering-arm64.cpp | 1438 ++ js/src/jit/arm64/Lowering-arm64.h | 135 + js/src/jit/arm64/MacroAssembler-arm64-inl.h | 4079 ++++ js/src/jit/arm64/MacroAssembler-arm64.cpp | 3416 ++++ js/src/jit/arm64/MacroAssembler-arm64.h | 2206 +++ js/src/jit/arm64/MoveEmitter-arm64.cpp | 329 + js/src/jit/arm64/MoveEmitter-arm64.h | 99 + js/src/jit/arm64/SharedICHelpers-arm64-inl.h | 79 + js/src/jit/arm64/SharedICHelpers-arm64.h | 82 + js/src/jit/arm64/SharedICRegisters-arm64.h | 51 + js/src/jit/arm64/Trampoline-arm64.cpp | 840 + js/src/jit/arm64/vixl/.clang-format | 4 + js/src/jit/arm64/vixl/AUTHORS | 8 + js/src/jit/arm64/vixl/Assembler-vixl.cpp | 5318 ++++++ js/src/jit/arm64/vixl/Assembler-vixl.h | 4974 +++++ js/src/jit/arm64/vixl/CompilerIntrinsics-vixl.h | 179 + js/src/jit/arm64/vixl/Constants-vixl.h | 2694 +++ js/src/jit/arm64/vixl/Cpu-Features-vixl.cpp | 231 + js/src/jit/arm64/vixl/Cpu-Features-vixl.h | 397 + js/src/jit/arm64/vixl/Cpu-vixl.cpp | 256 + js/src/jit/arm64/vixl/Cpu-vixl.h | 241 + js/src/jit/arm64/vixl/Debugger-vixl.cpp | 1535 ++ js/src/jit/arm64/vixl/Debugger-vixl.h | 117 + js/src/jit/arm64/vixl/Decoder-vixl.cpp | 899 + js/src/jit/arm64/vixl/Decoder-vixl.h | 276 + js/src/jit/arm64/vixl/Disasm-vixl.cpp | 3741 ++++ js/src/jit/arm64/vixl/Disasm-vixl.h | 181 + js/src/jit/arm64/vixl/Globals-vixl.h | 272 + js/src/jit/arm64/vixl/Instructions-vixl.cpp | 627 + js/src/jit/arm64/vixl/Instructions-vixl.h | 817 + js/src/jit/arm64/vixl/Instrument-vixl.cpp | 850 + js/src/jit/arm64/vixl/Instrument-vixl.h | 109 + js/src/jit/arm64/vixl/Logic-vixl.cpp | 4738 +++++ js/src/jit/arm64/vixl/MacroAssembler-vixl.cpp | 2027 ++ js/src/jit/arm64/vixl/MacroAssembler-vixl.h | 2622 +++ js/src/jit/arm64/vixl/MozAssembler-vixl.cpp | 610 + js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h | 356 + js/src/jit/arm64/vixl/MozCachingDecoder.h | 179 + js/src/jit/arm64/vixl/MozCpu-vixl.cpp | 226 + js/src/jit/arm64/vixl/MozInstructions-vixl.cpp | 211 + js/src/jit/arm64/vixl/MozSimulator-vixl.cpp | 1258 ++ js/src/jit/arm64/vixl/Platform-vixl.h | 39 + js/src/jit/arm64/vixl/README.md | 7 + js/src/jit/arm64/vixl/Simulator-Constants-vixl.h | 140 + js/src/jit/arm64/vixl/Simulator-vixl.cpp | 4371 +++++ js/src/jit/arm64/vixl/Simulator-vixl.h | 2592 +++ js/src/jit/arm64/vixl/Utils-vixl.cpp | 555 + js/src/jit/arm64/vixl/Utils-vixl.h | 1283 ++ js/src/jit/loong64/Architecture-loong64.cpp | 87 + js/src/jit/loong64/Architecture-loong64.h | 522 + js/src/jit/loong64/Assembler-loong64.cpp | 2478 +++ js/src/jit/loong64/Assembler-loong64.h | 1884 ++ js/src/jit/loong64/CodeGenerator-loong64.cpp | 2790 +++ js/src/jit/loong64/CodeGenerator-loong64.h | 209 + js/src/jit/loong64/LIR-loong64.h | 399 + js/src/jit/loong64/Lowering-loong64.cpp | 1088 ++ js/src/jit/loong64/Lowering-loong64.h | 110 + js/src/jit/loong64/MacroAssembler-loong64-inl.h | 2131 +++ js/src/jit/loong64/MacroAssembler-loong64.cpp | 5389 ++++++ js/src/jit/loong64/MacroAssembler-loong64.h | 1037 ++ js/src/jit/loong64/MoveEmitter-loong64.cpp | 326 + js/src/jit/loong64/MoveEmitter-loong64.h | 76 + js/src/jit/loong64/SharedICHelpers-loong64-inl.h | 83 + js/src/jit/loong64/SharedICHelpers-loong64.h | 91 + js/src/jit/loong64/SharedICRegisters-loong64.h | 42 + js/src/jit/loong64/Simulator-loong64.cpp | 5238 ++++++ js/src/jit/loong64/Simulator-loong64.h | 650 + js/src/jit/loong64/Trampoline-loong64.cpp | 833 + .../jit/mips-shared/Architecture-mips-shared.cpp | 121 + js/src/jit/mips-shared/Architecture-mips-shared.h | 341 + js/src/jit/mips-shared/Assembler-mips-shared.cpp | 2094 +++ js/src/jit/mips-shared/Assembler-mips-shared.h | 1500 ++ .../jit/mips-shared/AtomicOperations-mips-shared.h | 521 + js/src/jit/mips-shared/BaselineIC-mips-shared.cpp | 37 + .../jit/mips-shared/CodeGenerator-mips-shared.cpp | 2448 +++ js/src/jit/mips-shared/CodeGenerator-mips-shared.h | 157 + js/src/jit/mips-shared/LIR-mips-shared.h | 360 + js/src/jit/mips-shared/Lowering-mips-shared.cpp | 1024 + js/src/jit/mips-shared/Lowering-mips-shared.h | 89 + .../mips-shared/MacroAssembler-mips-shared-inl.h | 1307 ++ .../jit/mips-shared/MacroAssembler-mips-shared.cpp | 3355 ++++ .../jit/mips-shared/MacroAssembler-mips-shared.h | 258 + js/src/jit/mips-shared/MoveEmitter-mips-shared.cpp | 207 + js/src/jit/mips-shared/MoveEmitter-mips-shared.h | 73 + .../mips-shared/SharedICHelpers-mips-shared-inl.h | 82 + .../jit/mips-shared/SharedICHelpers-mips-shared.h | 88 + js/src/jit/mips32/Architecture-mips32.cpp | 94 + js/src/jit/mips32/Architecture-mips32.h | 282 + js/src/jit/mips32/Assembler-mips32.cpp | 369 + js/src/jit/mips32/Assembler-mips32.h | 265 + js/src/jit/mips32/CodeGenerator-mips32.cpp | 507 + js/src/jit/mips32/CodeGenerator-mips32.h | 60 + js/src/jit/mips32/LIR-mips32.h | 197 + js/src/jit/mips32/Lowering-mips32.cpp | 257 + js/src/jit/mips32/Lowering-mips32.h | 54 + js/src/jit/mips32/MacroAssembler-mips32-inl.h | 1027 + js/src/jit/mips32/MacroAssembler-mips32.cpp | 2825 +++ js/src/jit/mips32/MacroAssembler-mips32.h | 823 + js/src/jit/mips32/MoveEmitter-mips32.cpp | 152 + js/src/jit/mips32/MoveEmitter-mips32.h | 31 + js/src/jit/mips32/SharedICRegisters-mips32.h | 42 + js/src/jit/mips32/Simulator-mips32.cpp | 3629 ++++ js/src/jit/mips32/Simulator-mips32.h | 526 + js/src/jit/mips32/Trampoline-mips32.cpp | 942 + js/src/jit/mips64/Architecture-mips64.cpp | 88 + js/src/jit/mips64/Architecture-mips64.h | 233 + js/src/jit/mips64/Assembler-mips64.cpp | 371 + js/src/jit/mips64/Assembler-mips64.h | 288 + js/src/jit/mips64/CodeGenerator-mips64.cpp | 586 + js/src/jit/mips64/CodeGenerator-mips64.h | 65 + js/src/jit/mips64/LIR-mips64.h | 147 + js/src/jit/mips64/Lowering-mips64.cpp | 201 + js/src/jit/mips64/Lowering-mips64.h | 56 + js/src/jit/mips64/MacroAssembler-mips64-inl.h | 845 + js/src/jit/mips64/MacroAssembler-mips64.cpp | 2852 +++ js/src/jit/mips64/MacroAssembler-mips64.h | 841 + js/src/jit/mips64/MoveEmitter-mips64.cpp | 149 + js/src/jit/mips64/MoveEmitter-mips64.h | 31 + js/src/jit/mips64/SharedICRegisters-mips64.h | 45 + js/src/jit/mips64/Simulator-mips64.cpp | 4402 +++++ js/src/jit/mips64/Simulator-mips64.h | 536 + js/src/jit/mips64/Trampoline-mips64.cpp | 870 + js/src/jit/moz.build | 295 + js/src/jit/none/Architecture-none.h | 171 + js/src/jit/none/Assembler-none.h | 211 + js/src/jit/none/CodeGenerator-none.h | 78 + js/src/jit/none/LIR-none.h | 111 + js/src/jit/none/Lowering-none.h | 130 + js/src/jit/none/MacroAssembler-none.h | 454 + js/src/jit/none/MoveEmitter-none.h | 32 + js/src/jit/none/SharedICHelpers-none-inl.h | 31 + js/src/jit/none/SharedICHelpers-none.h | 32 + js/src/jit/none/SharedICRegisters-none.h | 32 + js/src/jit/none/Trampoline-none.cpp | 43 + js/src/jit/riscv64/Architecture-riscv64.cpp | 100 + js/src/jit/riscv64/Architecture-riscv64.h | 513 + js/src/jit/riscv64/Assembler-riscv64.cpp | 1548 ++ js/src/jit/riscv64/Assembler-riscv64.h | 685 + js/src/jit/riscv64/AssemblerMatInt.cpp | 217 + js/src/jit/riscv64/CodeGenerator-riscv64.cpp | 2871 +++ js/src/jit/riscv64/CodeGenerator-riscv64.h | 210 + js/src/jit/riscv64/LIR-riscv64.h | 399 + js/src/jit/riscv64/Lowering-riscv64.cpp | 1087 ++ js/src/jit/riscv64/Lowering-riscv64.h | 110 + js/src/jit/riscv64/MacroAssembler-riscv64-inl.h | 2025 ++ js/src/jit/riscv64/MacroAssembler-riscv64.cpp | 6515 +++++++ js/src/jit/riscv64/MacroAssembler-riscv64.h | 1224 ++ js/src/jit/riscv64/MoveEmitter-riscv64.cpp | 333 + js/src/jit/riscv64/MoveEmitter-riscv64.h | 70 + js/src/jit/riscv64/Register-riscv64.h | 186 + js/src/jit/riscv64/SharedICHelpers-riscv64-inl.h | 80 + js/src/jit/riscv64/SharedICHelpers-riscv64.h | 77 + js/src/jit/riscv64/SharedICRegisters-riscv64.h | 38 + js/src/jit/riscv64/Simulator-riscv64.cpp | 4718 +++++ js/src/jit/riscv64/Simulator-riscv64.h | 1281 ++ js/src/jit/riscv64/Trampoline-riscv64.cpp | 856 + .../jit/riscv64/constant/Base-constant-riscv.cpp | 247 + js/src/jit/riscv64/constant/Base-constant-riscv.h | 1057 ++ js/src/jit/riscv64/constant/Constant-riscv-a.h | 43 + js/src/jit/riscv64/constant/Constant-riscv-c.h | 61 + js/src/jit/riscv64/constant/Constant-riscv-d.h | 55 + js/src/jit/riscv64/constant/Constant-riscv-f.h | 51 + js/src/jit/riscv64/constant/Constant-riscv-i.h | 73 + js/src/jit/riscv64/constant/Constant-riscv-m.h | 34 + js/src/jit/riscv64/constant/Constant-riscv-v.h | 508 + js/src/jit/riscv64/constant/Constant-riscv-zicsr.h | 30 + .../jit/riscv64/constant/Constant-riscv-zifencei.h | 15 + js/src/jit/riscv64/constant/Constant-riscv64.h | 68 + js/src/jit/riscv64/constant/util-riscv64.h | 82 + js/src/jit/riscv64/disasm/Disasm-riscv64.cpp | 2155 +++ js/src/jit/riscv64/disasm/Disasm-riscv64.h | 74 + .../jit/riscv64/extension/base-assembler-riscv.cc | 517 + .../jit/riscv64/extension/base-assembler-riscv.h | 219 + js/src/jit/riscv64/extension/base-riscv-i.cc | 351 + js/src/jit/riscv64/extension/base-riscv-i.h | 273 + js/src/jit/riscv64/extension/extension-riscv-a.cc | 123 + js/src/jit/riscv64/extension/extension-riscv-a.h | 46 + js/src/jit/riscv64/extension/extension-riscv-c.cc | 275 + js/src/jit/riscv64/extension/extension-riscv-c.h | 77 + js/src/jit/riscv64/extension/extension-riscv-d.cc | 167 + js/src/jit/riscv64/extension/extension-riscv-d.h | 68 + js/src/jit/riscv64/extension/extension-riscv-f.cc | 158 + js/src/jit/riscv64/extension/extension-riscv-f.h | 66 + js/src/jit/riscv64/extension/extension-riscv-m.cc | 68 + js/src/jit/riscv64/extension/extension-riscv-m.h | 37 + js/src/jit/riscv64/extension/extension-riscv-v.cc | 891 + js/src/jit/riscv64/extension/extension-riscv-v.h | 484 + .../jit/riscv64/extension/extension-riscv-zicsr.cc | 44 + .../jit/riscv64/extension/extension-riscv-zicsr.h | 57 + .../riscv64/extension/extension-riscv-zifencei.cc | 17 + .../riscv64/extension/extension-riscv-zifencei.h | 20 + js/src/jit/shared/Architecture-shared.h | 18 + js/src/jit/shared/Assembler-shared.cpp | 74 + js/src/jit/shared/Assembler-shared.h | 716 + .../shared/AtomicOperations-feeling-lucky-gcc.h | 453 + js/src/jit/shared/AtomicOperations-feeling-lucky.h | 16 + js/src/jit/shared/AtomicOperations-shared-jit.cpp | 180 + js/src/jit/shared/AtomicOperations-shared-jit.h | 490 + js/src/jit/shared/CodeGenerator-shared-inl.h | 342 + js/src/jit/shared/CodeGenerator-shared.cpp | 983 + js/src/jit/shared/CodeGenerator-shared.h | 488 + js/src/jit/shared/Disassembler-shared.cpp | 248 + js/src/jit/shared/Disassembler-shared.h | 184 + js/src/jit/shared/IonAssemblerBuffer.h | 438 + .../shared/IonAssemblerBufferWithConstantPools.h | 1197 ++ js/src/jit/shared/LIR-shared.h | 4272 +++++ js/src/jit/shared/Lowering-shared-inl.h | 894 + js/src/jit/shared/Lowering-shared.cpp | 319 + js/src/jit/shared/Lowering-shared.h | 371 + js/src/jit/wasm32/Architecture-wasm32.h | 174 + js/src/jit/wasm32/Assembler-wasm32.h | 229 + js/src/jit/wasm32/CodeGenerator-wasm32.cpp | 254 + js/src/jit/wasm32/CodeGenerator-wasm32.h | 76 + js/src/jit/wasm32/LIR-wasm32.h | 109 + js/src/jit/wasm32/Lowering-wasm32.h | 128 + js/src/jit/wasm32/MacroAssembler-wasm32-inl.h | 1176 ++ js/src/jit/wasm32/MacroAssembler-wasm32.cpp | 502 + js/src/jit/wasm32/MacroAssembler-wasm32.h | 528 + js/src/jit/wasm32/MoveEmitter-wasm32.h | 30 + js/src/jit/wasm32/SharedICHelpers-wasm32-inl.h | 32 + js/src/jit/wasm32/SharedICHelpers-wasm32.h | 30 + js/src/jit/wasm32/SharedICRegisters-wasm32.h | 36 + js/src/jit/wasm32/Trampoline-wasm32.cpp | 46 + js/src/jit/x64/Assembler-x64.cpp | 246 + js/src/jit/x64/Assembler-x64.h | 1249 ++ js/src/jit/x64/BaseAssembler-x64.h | 1373 ++ js/src/jit/x64/CodeGenerator-x64.cpp | 984 + js/src/jit/x64/CodeGenerator-x64.h | 41 + js/src/jit/x64/LIR-x64.h | 170 + js/src/jit/x64/Lowering-x64.cpp | 565 + js/src/jit/x64/Lowering-x64.h | 70 + js/src/jit/x64/MacroAssembler-x64-inl.h | 1099 ++ js/src/jit/x64/MacroAssembler-x64.cpp | 1747 ++ js/src/jit/x64/MacroAssembler-x64.h | 1218 ++ js/src/jit/x64/SharedICHelpers-x64-inl.h | 80 + js/src/jit/x64/SharedICHelpers-x64.h | 70 + js/src/jit/x64/SharedICRegisters-x64.h | 33 + js/src/jit/x64/Trampoline-x64.cpp | 888 + js/src/jit/x86-shared/Architecture-x86-shared.cpp | 93 + js/src/jit/x86-shared/Architecture-x86-shared.h | 467 + js/src/jit/x86-shared/Assembler-x86-shared.cpp | 355 + js/src/jit/x86-shared/Assembler-x86-shared.h | 4887 +++++ .../jit/x86-shared/AssemblerBuffer-x86-shared.cpp | 57 + js/src/jit/x86-shared/AssemblerBuffer-x86-shared.h | 256 + js/src/jit/x86-shared/BaseAssembler-x86-shared.h | 6460 +++++++ js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp | 3883 ++++ js/src/jit/x86-shared/CodeGenerator-x86-shared.h | 189 + js/src/jit/x86-shared/Constants-x86-shared.h | 326 + js/src/jit/x86-shared/Encoding-x86-shared.h | 508 + js/src/jit/x86-shared/LIR-x86-shared.h | 304 + js/src/jit/x86-shared/Lowering-x86-shared.cpp | 1863 ++ js/src/jit/x86-shared/Lowering-x86-shared.h | 78 + .../x86-shared/MacroAssembler-x86-shared-SIMD.cpp | 1484 ++ .../jit/x86-shared/MacroAssembler-x86-shared-inl.h | 3396 ++++ .../jit/x86-shared/MacroAssembler-x86-shared.cpp | 2132 +++ js/src/jit/x86-shared/MacroAssembler-x86-shared.h | 998 + js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp | 528 + js/src/jit/x86-shared/MoveEmitter-x86-shared.h | 83 + js/src/jit/x86-shared/Patching-x86-shared.h | 113 + js/src/jit/x86/Assembler-x86.cpp | 85 + js/src/jit/x86/Assembler-x86.h | 1079 ++ js/src/jit/x86/BaseAssembler-x86.h | 190 + js/src/jit/x86/CodeGenerator-x86.cpp | 1509 ++ js/src/jit/x86/CodeGenerator-x86.h | 49 + js/src/jit/x86/LIR-x86.h | 308 + js/src/jit/x86/Lowering-x86.cpp | 840 + js/src/jit/x86/Lowering-x86.h | 79 + js/src/jit/x86/MacroAssembler-x86-inl.h | 1386 ++ js/src/jit/x86/MacroAssembler-x86.cpp | 1829 ++ js/src/jit/x86/MacroAssembler-x86.h | 1149 ++ js/src/jit/x86/SharedICHelpers-x86-inl.h | 77 + js/src/jit/x86/SharedICHelpers-x86.h | 70 + js/src/jit/x86/SharedICRegisters-x86.h | 36 + js/src/jit/x86/Trampoline-x86.cpp | 796 + 531 files changed, 466249 insertions(+) create mode 100644 js/src/jit/ABIArgGenerator.h create mode 100644 js/src/jit/ABIFunctionList-inl.h create mode 100644 js/src/jit/ABIFunctions.h create mode 100644 js/src/jit/AliasAnalysis.cpp create mode 100644 js/src/jit/AliasAnalysis.h create mode 100644 js/src/jit/AlignmentMaskAnalysis.cpp create mode 100644 js/src/jit/AlignmentMaskAnalysis.h create mode 100644 js/src/jit/Assembler.h create mode 100644 js/src/jit/AtomicOp.h create mode 100644 js/src/jit/AtomicOperations.h create mode 100644 js/src/jit/AutoWritableJitCode.h create mode 100644 js/src/jit/BacktrackingAllocator.cpp create mode 100644 js/src/jit/BacktrackingAllocator.h create mode 100644 js/src/jit/Bailouts.cpp create mode 100644 js/src/jit/Bailouts.h create mode 100644 js/src/jit/BaselineBailouts.cpp create mode 100644 js/src/jit/BaselineCacheIRCompiler.cpp create mode 100644 js/src/jit/BaselineCacheIRCompiler.h create mode 100644 js/src/jit/BaselineCodeGen.cpp create mode 100644 js/src/jit/BaselineCodeGen.h create mode 100644 js/src/jit/BaselineDebugModeOSR.cpp create mode 100644 js/src/jit/BaselineDebugModeOSR.h create mode 100644 js/src/jit/BaselineFrame-inl.h create mode 100644 js/src/jit/BaselineFrame.cpp create mode 100644 js/src/jit/BaselineFrame.h create mode 100644 js/src/jit/BaselineFrameInfo-inl.h create mode 100644 js/src/jit/BaselineFrameInfo.cpp create mode 100644 js/src/jit/BaselineFrameInfo.h create mode 100644 js/src/jit/BaselineIC.cpp create mode 100644 js/src/jit/BaselineIC.h create mode 100644 js/src/jit/BaselineICList.h create mode 100644 js/src/jit/BaselineJIT.cpp create mode 100644 js/src/jit/BaselineJIT.h create mode 100644 js/src/jit/BitSet.cpp create mode 100644 js/src/jit/BitSet.h create mode 100644 js/src/jit/BytecodeAnalysis.cpp create mode 100644 js/src/jit/BytecodeAnalysis.h create mode 100644 js/src/jit/CacheIR.cpp create mode 100644 js/src/jit/CacheIR.h create mode 100644 js/src/jit/CacheIRCloner.h create mode 100644 js/src/jit/CacheIRCompiler.cpp create mode 100644 js/src/jit/CacheIRCompiler.h create mode 100644 js/src/jit/CacheIRGenerator.h create mode 100644 js/src/jit/CacheIRHealth.cpp create mode 100644 js/src/jit/CacheIRHealth.h create mode 100644 js/src/jit/CacheIROps.yaml create mode 100644 js/src/jit/CacheIRReader.h create mode 100644 js/src/jit/CacheIRSpewer.cpp create mode 100644 js/src/jit/CacheIRSpewer.h create mode 100644 js/src/jit/CacheIRWriter.h create mode 100644 js/src/jit/CalleeToken.h create mode 100644 js/src/jit/CodeGenerator.cpp create mode 100644 js/src/jit/CodeGenerator.h create mode 100644 js/src/jit/CompactBuffer.h create mode 100644 js/src/jit/CompileInfo.h create mode 100644 js/src/jit/CompileWrappers.cpp create mode 100644 js/src/jit/CompileWrappers.h create mode 100644 js/src/jit/Disassemble.cpp create mode 100644 js/src/jit/Disassemble.h create mode 100644 js/src/jit/EdgeCaseAnalysis.cpp create mode 100644 js/src/jit/EdgeCaseAnalysis.h create mode 100644 js/src/jit/EffectiveAddressAnalysis.cpp create mode 100644 js/src/jit/EffectiveAddressAnalysis.h create mode 100644 js/src/jit/ExecutableAllocator.cpp create mode 100644 js/src/jit/ExecutableAllocator.h create mode 100644 js/src/jit/FixedList.h create mode 100644 js/src/jit/FlushICache.cpp create mode 100644 js/src/jit/FlushICache.h create mode 100644 js/src/jit/FoldLinearArithConstants.cpp create mode 100644 js/src/jit/FoldLinearArithConstants.h create mode 100644 js/src/jit/GenerateAtomicOperations.py create mode 100644 js/src/jit/GenerateCacheIRFiles.py create mode 100644 js/src/jit/GenerateLIRFiles.py create mode 100644 js/src/jit/GenerateMIRFiles.py create mode 100644 js/src/jit/ICState.h create mode 100644 js/src/jit/ICStubSpace.h create mode 100644 js/src/jit/InlinableNatives.cpp create mode 100644 js/src/jit/InlinableNatives.h create mode 100644 js/src/jit/InlineList.h create mode 100644 js/src/jit/InlineScriptTree-inl.h create mode 100644 js/src/jit/InlineScriptTree.h create mode 100644 js/src/jit/InstructionReordering.cpp create mode 100644 js/src/jit/InstructionReordering.h create mode 100644 js/src/jit/InterpreterEntryTrampoline.cpp create mode 100644 js/src/jit/InterpreterEntryTrampoline.h create mode 100644 js/src/jit/Invalidation.h create mode 100644 js/src/jit/Ion.cpp create mode 100644 js/src/jit/Ion.h create mode 100644 js/src/jit/IonAnalysis.cpp create mode 100644 js/src/jit/IonAnalysis.h create mode 100644 js/src/jit/IonCacheIRCompiler.cpp create mode 100644 js/src/jit/IonCacheIRCompiler.h create mode 100644 js/src/jit/IonCompileTask.cpp create mode 100644 js/src/jit/IonCompileTask.h create mode 100644 js/src/jit/IonIC.cpp create mode 100644 js/src/jit/IonIC.h create mode 100644 js/src/jit/IonOptimizationLevels.cpp create mode 100644 js/src/jit/IonOptimizationLevels.h create mode 100644 js/src/jit/IonScript.h create mode 100644 js/src/jit/IonTypes.h create mode 100644 js/src/jit/JSJitFrameIter-inl.h create mode 100644 js/src/jit/JSJitFrameIter.cpp create mode 100644 js/src/jit/JSJitFrameIter.h create mode 100644 js/src/jit/JSONSpewer.cpp create mode 100644 js/src/jit/JSONSpewer.h create mode 100644 js/src/jit/Jit.cpp create mode 100644 js/src/jit/Jit.h create mode 100644 js/src/jit/JitAllocPolicy.h create mode 100644 js/src/jit/JitCode.h create mode 100644 js/src/jit/JitCommon.h create mode 100644 js/src/jit/JitContext.cpp create mode 100644 js/src/jit/JitContext.h create mode 100644 js/src/jit/JitFrames-inl.h create mode 100644 js/src/jit/JitFrames.cpp create mode 100644 js/src/jit/JitFrames.h create mode 100644 js/src/jit/JitHints-inl.h create mode 100644 js/src/jit/JitHints.h create mode 100644 js/src/jit/JitOptions.cpp create mode 100644 js/src/jit/JitOptions.h create mode 100644 js/src/jit/JitRealm.h create mode 100644 js/src/jit/JitRuntime.h create mode 100644 js/src/jit/JitScript-inl.h create mode 100644 js/src/jit/JitScript.cpp create mode 100644 js/src/jit/JitScript.h create mode 100644 js/src/jit/JitSpewer.cpp create mode 100644 js/src/jit/JitSpewer.h create mode 100644 js/src/jit/JitZone.h create mode 100644 js/src/jit/JitcodeMap.cpp create mode 100644 js/src/jit/JitcodeMap.h create mode 100644 js/src/jit/Jitdump.h create mode 100644 js/src/jit/KnownClass.cpp create mode 100644 js/src/jit/KnownClass.h create mode 100644 js/src/jit/LICM.cpp create mode 100644 js/src/jit/LICM.h create mode 100644 js/src/jit/LIR.cpp create mode 100644 js/src/jit/LIR.h create mode 100644 js/src/jit/LIROps.yaml create mode 100644 js/src/jit/Label.cpp create mode 100644 js/src/jit/Label.h create mode 100644 js/src/jit/Linker.cpp create mode 100644 js/src/jit/Linker.h create mode 100644 js/src/jit/Lowering.cpp create mode 100644 js/src/jit/Lowering.h create mode 100644 js/src/jit/MIR.cpp create mode 100644 js/src/jit/MIR.h create mode 100644 js/src/jit/MIRGenerator.h create mode 100644 js/src/jit/MIRGraph.cpp create mode 100644 js/src/jit/MIRGraph.h create mode 100644 js/src/jit/MIROps.yaml create mode 100644 js/src/jit/MachineState.h create mode 100644 js/src/jit/MacroAssembler-inl.h create mode 100644 js/src/jit/MacroAssembler.cpp create mode 100644 js/src/jit/MacroAssembler.h create mode 100644 js/src/jit/MoveEmitter.h create mode 100644 js/src/jit/MoveResolver.cpp create mode 100644 js/src/jit/MoveResolver.h create mode 100644 js/src/jit/PcScriptCache.h create mode 100644 js/src/jit/PerfSpewer.cpp create mode 100644 js/src/jit/PerfSpewer.h create mode 100644 js/src/jit/ProcessExecutableMemory.cpp create mode 100644 js/src/jit/ProcessExecutableMemory.h create mode 100644 js/src/jit/RangeAnalysis.cpp create mode 100644 js/src/jit/RangeAnalysis.h create mode 100644 js/src/jit/ReciprocalMulConstants.cpp create mode 100644 js/src/jit/ReciprocalMulConstants.h create mode 100644 js/src/jit/Recover.cpp create mode 100644 js/src/jit/Recover.h create mode 100644 js/src/jit/RegExpStubConstants.h create mode 100644 js/src/jit/RegisterAllocator.cpp create mode 100644 js/src/jit/RegisterAllocator.h create mode 100644 js/src/jit/RegisterSets.h create mode 100644 js/src/jit/Registers.h create mode 100644 js/src/jit/RematerializedFrame-inl.h create mode 100644 js/src/jit/RematerializedFrame.cpp create mode 100644 js/src/jit/RematerializedFrame.h create mode 100644 js/src/jit/SafepointIndex-inl.h create mode 100644 js/src/jit/SafepointIndex.cpp create mode 100644 js/src/jit/SafepointIndex.h create mode 100644 js/src/jit/Safepoints.cpp create mode 100644 js/src/jit/Safepoints.h create mode 100644 js/src/jit/ScalarReplacement.cpp create mode 100644 js/src/jit/ScalarReplacement.h create mode 100644 js/src/jit/ScalarTypeUtils.h create mode 100644 js/src/jit/ScriptFromCalleeToken.h create mode 100644 js/src/jit/SharedICHelpers-inl.h create mode 100644 js/src/jit/SharedICHelpers.h create mode 100644 js/src/jit/SharedICRegisters.h create mode 100644 js/src/jit/ShuffleAnalysis.cpp create mode 100644 js/src/jit/ShuffleAnalysis.h create mode 100644 js/src/jit/Simulator.h create mode 100644 js/src/jit/Sink.cpp create mode 100644 js/src/jit/Sink.h create mode 100644 js/src/jit/Snapshots.cpp create mode 100644 js/src/jit/Snapshots.h create mode 100644 js/src/jit/StackSlotAllocator.h create mode 100644 js/src/jit/TemplateObject-inl.h create mode 100644 js/src/jit/TemplateObject.h create mode 100644 js/src/jit/Trampoline.cpp create mode 100644 js/src/jit/TrialInlining.cpp create mode 100644 js/src/jit/TrialInlining.h create mode 100644 js/src/jit/TypeData.h create mode 100644 js/src/jit/TypePolicy.cpp create mode 100644 js/src/jit/TypePolicy.h create mode 100644 js/src/jit/VMFunctionList-inl.h create mode 100644 js/src/jit/VMFunctions.cpp create mode 100644 js/src/jit/VMFunctions.h create mode 100644 js/src/jit/ValueNumbering.cpp create mode 100644 js/src/jit/ValueNumbering.h create mode 100644 js/src/jit/WarpBuilder.cpp create mode 100644 js/src/jit/WarpBuilder.h create mode 100644 js/src/jit/WarpBuilderShared.cpp create mode 100644 js/src/jit/WarpBuilderShared.h create mode 100644 js/src/jit/WarpCacheIRTranspiler.cpp create mode 100644 js/src/jit/WarpCacheIRTranspiler.h create mode 100644 js/src/jit/WarpOracle.cpp create mode 100644 js/src/jit/WarpOracle.h create mode 100644 js/src/jit/WarpSnapshot.cpp create mode 100644 js/src/jit/WarpSnapshot.h create mode 100644 js/src/jit/WasmBCE.cpp create mode 100644 js/src/jit/WasmBCE.h create mode 100644 js/src/jit/XrayJitInfo.cpp create mode 100644 js/src/jit/arm/Architecture-arm.cpp create mode 100644 js/src/jit/arm/Architecture-arm.h create mode 100644 js/src/jit/arm/Assembler-arm.cpp create mode 100644 js/src/jit/arm/Assembler-arm.h create mode 100644 js/src/jit/arm/CodeGenerator-arm.cpp create mode 100644 js/src/jit/arm/CodeGenerator-arm.h create mode 100644 js/src/jit/arm/DoubleEntryTable.tbl create mode 100644 js/src/jit/arm/LIR-arm.h create mode 100644 js/src/jit/arm/Lowering-arm.cpp create mode 100644 js/src/jit/arm/Lowering-arm.h create mode 100644 js/src/jit/arm/MacroAssembler-arm-inl.h create mode 100644 js/src/jit/arm/MacroAssembler-arm.cpp create mode 100644 js/src/jit/arm/MacroAssembler-arm.h create mode 100644 js/src/jit/arm/MoveEmitter-arm.cpp create mode 100644 js/src/jit/arm/MoveEmitter-arm.h create mode 100644 js/src/jit/arm/SharedICHelpers-arm-inl.h create mode 100644 js/src/jit/arm/SharedICHelpers-arm.h create mode 100644 js/src/jit/arm/SharedICRegisters-arm.h create mode 100644 js/src/jit/arm/Simulator-arm.cpp create mode 100644 js/src/jit/arm/Simulator-arm.h create mode 100644 js/src/jit/arm/Trampoline-arm.cpp create mode 100644 js/src/jit/arm/disasm/Constants-arm.cpp create mode 100644 js/src/jit/arm/disasm/Constants-arm.h create mode 100644 js/src/jit/arm/disasm/Disasm-arm.cpp create mode 100644 js/src/jit/arm/disasm/Disasm-arm.h create mode 100644 js/src/jit/arm/gen-double-encoder-table.py create mode 100644 js/src/jit/arm/llvm-compiler-rt/arm/aeabi_idivmod.S create mode 100644 js/src/jit/arm/llvm-compiler-rt/arm/aeabi_uidivmod.S create mode 100644 js/src/jit/arm/llvm-compiler-rt/assembly.h create mode 100644 js/src/jit/arm64/Architecture-arm64.cpp create mode 100644 js/src/jit/arm64/Architecture-arm64.h create mode 100644 js/src/jit/arm64/Assembler-arm64.cpp create mode 100644 js/src/jit/arm64/Assembler-arm64.h create mode 100644 js/src/jit/arm64/CodeGenerator-arm64.cpp create mode 100644 js/src/jit/arm64/CodeGenerator-arm64.h create mode 100644 js/src/jit/arm64/LIR-arm64.h create mode 100644 js/src/jit/arm64/Lowering-arm64.cpp create mode 100644 js/src/jit/arm64/Lowering-arm64.h create mode 100644 js/src/jit/arm64/MacroAssembler-arm64-inl.h create mode 100644 js/src/jit/arm64/MacroAssembler-arm64.cpp create mode 100644 js/src/jit/arm64/MacroAssembler-arm64.h create mode 100644 js/src/jit/arm64/MoveEmitter-arm64.cpp create mode 100644 js/src/jit/arm64/MoveEmitter-arm64.h create mode 100644 js/src/jit/arm64/SharedICHelpers-arm64-inl.h create mode 100644 js/src/jit/arm64/SharedICHelpers-arm64.h create mode 100644 js/src/jit/arm64/SharedICRegisters-arm64.h create mode 100644 js/src/jit/arm64/Trampoline-arm64.cpp create mode 100644 js/src/jit/arm64/vixl/.clang-format create mode 100644 js/src/jit/arm64/vixl/AUTHORS create mode 100644 js/src/jit/arm64/vixl/Assembler-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Assembler-vixl.h create mode 100644 js/src/jit/arm64/vixl/CompilerIntrinsics-vixl.h create mode 100644 js/src/jit/arm64/vixl/Constants-vixl.h create mode 100644 js/src/jit/arm64/vixl/Cpu-Features-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Cpu-Features-vixl.h create mode 100644 js/src/jit/arm64/vixl/Cpu-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Cpu-vixl.h create mode 100644 js/src/jit/arm64/vixl/Debugger-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Debugger-vixl.h create mode 100644 js/src/jit/arm64/vixl/Decoder-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Decoder-vixl.h create mode 100644 js/src/jit/arm64/vixl/Disasm-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Disasm-vixl.h create mode 100644 js/src/jit/arm64/vixl/Globals-vixl.h create mode 100644 js/src/jit/arm64/vixl/Instructions-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Instructions-vixl.h create mode 100644 js/src/jit/arm64/vixl/Instrument-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Instrument-vixl.h create mode 100644 js/src/jit/arm64/vixl/Logic-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/MacroAssembler-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/MacroAssembler-vixl.h create mode 100644 js/src/jit/arm64/vixl/MozAssembler-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h create mode 100644 js/src/jit/arm64/vixl/MozCachingDecoder.h create mode 100644 js/src/jit/arm64/vixl/MozCpu-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/MozInstructions-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/MozSimulator-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Platform-vixl.h create mode 100644 js/src/jit/arm64/vixl/README.md create mode 100644 js/src/jit/arm64/vixl/Simulator-Constants-vixl.h create mode 100644 js/src/jit/arm64/vixl/Simulator-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Simulator-vixl.h create mode 100644 js/src/jit/arm64/vixl/Utils-vixl.cpp create mode 100644 js/src/jit/arm64/vixl/Utils-vixl.h create mode 100644 js/src/jit/loong64/Architecture-loong64.cpp create mode 100644 js/src/jit/loong64/Architecture-loong64.h create mode 100644 js/src/jit/loong64/Assembler-loong64.cpp create mode 100644 js/src/jit/loong64/Assembler-loong64.h create mode 100644 js/src/jit/loong64/CodeGenerator-loong64.cpp create mode 100644 js/src/jit/loong64/CodeGenerator-loong64.h create mode 100644 js/src/jit/loong64/LIR-loong64.h create mode 100644 js/src/jit/loong64/Lowering-loong64.cpp create mode 100644 js/src/jit/loong64/Lowering-loong64.h create mode 100644 js/src/jit/loong64/MacroAssembler-loong64-inl.h create mode 100644 js/src/jit/loong64/MacroAssembler-loong64.cpp create mode 100644 js/src/jit/loong64/MacroAssembler-loong64.h create mode 100644 js/src/jit/loong64/MoveEmitter-loong64.cpp create mode 100644 js/src/jit/loong64/MoveEmitter-loong64.h create mode 100644 js/src/jit/loong64/SharedICHelpers-loong64-inl.h create mode 100644 js/src/jit/loong64/SharedICHelpers-loong64.h create mode 100644 js/src/jit/loong64/SharedICRegisters-loong64.h create mode 100644 js/src/jit/loong64/Simulator-loong64.cpp create mode 100644 js/src/jit/loong64/Simulator-loong64.h create mode 100644 js/src/jit/loong64/Trampoline-loong64.cpp create mode 100644 js/src/jit/mips-shared/Architecture-mips-shared.cpp create mode 100644 js/src/jit/mips-shared/Architecture-mips-shared.h create mode 100644 js/src/jit/mips-shared/Assembler-mips-shared.cpp create mode 100644 js/src/jit/mips-shared/Assembler-mips-shared.h create mode 100644 js/src/jit/mips-shared/AtomicOperations-mips-shared.h create mode 100644 js/src/jit/mips-shared/BaselineIC-mips-shared.cpp create mode 100644 js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp create mode 100644 js/src/jit/mips-shared/CodeGenerator-mips-shared.h create mode 100644 js/src/jit/mips-shared/LIR-mips-shared.h create mode 100644 js/src/jit/mips-shared/Lowering-mips-shared.cpp create mode 100644 js/src/jit/mips-shared/Lowering-mips-shared.h create mode 100644 js/src/jit/mips-shared/MacroAssembler-mips-shared-inl.h create mode 100644 js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp create mode 100644 js/src/jit/mips-shared/MacroAssembler-mips-shared.h create mode 100644 js/src/jit/mips-shared/MoveEmitter-mips-shared.cpp create mode 100644 js/src/jit/mips-shared/MoveEmitter-mips-shared.h create mode 100644 js/src/jit/mips-shared/SharedICHelpers-mips-shared-inl.h create mode 100644 js/src/jit/mips-shared/SharedICHelpers-mips-shared.h create mode 100644 js/src/jit/mips32/Architecture-mips32.cpp create mode 100644 js/src/jit/mips32/Architecture-mips32.h create mode 100644 js/src/jit/mips32/Assembler-mips32.cpp create mode 100644 js/src/jit/mips32/Assembler-mips32.h create mode 100644 js/src/jit/mips32/CodeGenerator-mips32.cpp create mode 100644 js/src/jit/mips32/CodeGenerator-mips32.h create mode 100644 js/src/jit/mips32/LIR-mips32.h create mode 100644 js/src/jit/mips32/Lowering-mips32.cpp create mode 100644 js/src/jit/mips32/Lowering-mips32.h create mode 100644 js/src/jit/mips32/MacroAssembler-mips32-inl.h create mode 100644 js/src/jit/mips32/MacroAssembler-mips32.cpp create mode 100644 js/src/jit/mips32/MacroAssembler-mips32.h create mode 100644 js/src/jit/mips32/MoveEmitter-mips32.cpp create mode 100644 js/src/jit/mips32/MoveEmitter-mips32.h create mode 100644 js/src/jit/mips32/SharedICRegisters-mips32.h create mode 100644 js/src/jit/mips32/Simulator-mips32.cpp create mode 100644 js/src/jit/mips32/Simulator-mips32.h create mode 100644 js/src/jit/mips32/Trampoline-mips32.cpp create mode 100644 js/src/jit/mips64/Architecture-mips64.cpp create mode 100644 js/src/jit/mips64/Architecture-mips64.h create mode 100644 js/src/jit/mips64/Assembler-mips64.cpp create mode 100644 js/src/jit/mips64/Assembler-mips64.h create mode 100644 js/src/jit/mips64/CodeGenerator-mips64.cpp create mode 100644 js/src/jit/mips64/CodeGenerator-mips64.h create mode 100644 js/src/jit/mips64/LIR-mips64.h create mode 100644 js/src/jit/mips64/Lowering-mips64.cpp create mode 100644 js/src/jit/mips64/Lowering-mips64.h create mode 100644 js/src/jit/mips64/MacroAssembler-mips64-inl.h create mode 100644 js/src/jit/mips64/MacroAssembler-mips64.cpp create mode 100644 js/src/jit/mips64/MacroAssembler-mips64.h create mode 100644 js/src/jit/mips64/MoveEmitter-mips64.cpp create mode 100644 js/src/jit/mips64/MoveEmitter-mips64.h create mode 100644 js/src/jit/mips64/SharedICRegisters-mips64.h create mode 100644 js/src/jit/mips64/Simulator-mips64.cpp create mode 100644 js/src/jit/mips64/Simulator-mips64.h create mode 100644 js/src/jit/mips64/Trampoline-mips64.cpp create mode 100644 js/src/jit/moz.build create mode 100644 js/src/jit/none/Architecture-none.h create mode 100644 js/src/jit/none/Assembler-none.h create mode 100644 js/src/jit/none/CodeGenerator-none.h create mode 100644 js/src/jit/none/LIR-none.h create mode 100644 js/src/jit/none/Lowering-none.h create mode 100644 js/src/jit/none/MacroAssembler-none.h create mode 100644 js/src/jit/none/MoveEmitter-none.h create mode 100644 js/src/jit/none/SharedICHelpers-none-inl.h create mode 100644 js/src/jit/none/SharedICHelpers-none.h create mode 100644 js/src/jit/none/SharedICRegisters-none.h create mode 100644 js/src/jit/none/Trampoline-none.cpp create mode 100644 js/src/jit/riscv64/Architecture-riscv64.cpp create mode 100644 js/src/jit/riscv64/Architecture-riscv64.h create mode 100644 js/src/jit/riscv64/Assembler-riscv64.cpp create mode 100644 js/src/jit/riscv64/Assembler-riscv64.h create mode 100644 js/src/jit/riscv64/AssemblerMatInt.cpp create mode 100644 js/src/jit/riscv64/CodeGenerator-riscv64.cpp create mode 100644 js/src/jit/riscv64/CodeGenerator-riscv64.h create mode 100644 js/src/jit/riscv64/LIR-riscv64.h create mode 100644 js/src/jit/riscv64/Lowering-riscv64.cpp create mode 100644 js/src/jit/riscv64/Lowering-riscv64.h create mode 100644 js/src/jit/riscv64/MacroAssembler-riscv64-inl.h create mode 100644 js/src/jit/riscv64/MacroAssembler-riscv64.cpp create mode 100644 js/src/jit/riscv64/MacroAssembler-riscv64.h create mode 100644 js/src/jit/riscv64/MoveEmitter-riscv64.cpp create mode 100644 js/src/jit/riscv64/MoveEmitter-riscv64.h create mode 100644 js/src/jit/riscv64/Register-riscv64.h create mode 100644 js/src/jit/riscv64/SharedICHelpers-riscv64-inl.h create mode 100644 js/src/jit/riscv64/SharedICHelpers-riscv64.h create mode 100644 js/src/jit/riscv64/SharedICRegisters-riscv64.h create mode 100644 js/src/jit/riscv64/Simulator-riscv64.cpp create mode 100644 js/src/jit/riscv64/Simulator-riscv64.h create mode 100644 js/src/jit/riscv64/Trampoline-riscv64.cpp create mode 100644 js/src/jit/riscv64/constant/Base-constant-riscv.cpp create mode 100644 js/src/jit/riscv64/constant/Base-constant-riscv.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv-a.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv-c.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv-d.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv-f.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv-i.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv-m.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv-v.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv-zicsr.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv-zifencei.h create mode 100644 js/src/jit/riscv64/constant/Constant-riscv64.h create mode 100644 js/src/jit/riscv64/constant/util-riscv64.h create mode 100644 js/src/jit/riscv64/disasm/Disasm-riscv64.cpp create mode 100644 js/src/jit/riscv64/disasm/Disasm-riscv64.h create mode 100644 js/src/jit/riscv64/extension/base-assembler-riscv.cc create mode 100644 js/src/jit/riscv64/extension/base-assembler-riscv.h create mode 100644 js/src/jit/riscv64/extension/base-riscv-i.cc create mode 100644 js/src/jit/riscv64/extension/base-riscv-i.h create mode 100644 js/src/jit/riscv64/extension/extension-riscv-a.cc create mode 100644 js/src/jit/riscv64/extension/extension-riscv-a.h create mode 100644 js/src/jit/riscv64/extension/extension-riscv-c.cc create mode 100644 js/src/jit/riscv64/extension/extension-riscv-c.h create mode 100644 js/src/jit/riscv64/extension/extension-riscv-d.cc create mode 100644 js/src/jit/riscv64/extension/extension-riscv-d.h create mode 100644 js/src/jit/riscv64/extension/extension-riscv-f.cc create mode 100644 js/src/jit/riscv64/extension/extension-riscv-f.h create mode 100644 js/src/jit/riscv64/extension/extension-riscv-m.cc create mode 100644 js/src/jit/riscv64/extension/extension-riscv-m.h create mode 100644 js/src/jit/riscv64/extension/extension-riscv-v.cc create mode 100644 js/src/jit/riscv64/extension/extension-riscv-v.h create mode 100644 js/src/jit/riscv64/extension/extension-riscv-zicsr.cc create mode 100644 js/src/jit/riscv64/extension/extension-riscv-zicsr.h create mode 100644 js/src/jit/riscv64/extension/extension-riscv-zifencei.cc create mode 100644 js/src/jit/riscv64/extension/extension-riscv-zifencei.h create mode 100644 js/src/jit/shared/Architecture-shared.h create mode 100644 js/src/jit/shared/Assembler-shared.cpp create mode 100644 js/src/jit/shared/Assembler-shared.h create mode 100644 js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h create mode 100644 js/src/jit/shared/AtomicOperations-feeling-lucky.h create mode 100644 js/src/jit/shared/AtomicOperations-shared-jit.cpp create mode 100644 js/src/jit/shared/AtomicOperations-shared-jit.h create mode 100644 js/src/jit/shared/CodeGenerator-shared-inl.h create mode 100644 js/src/jit/shared/CodeGenerator-shared.cpp create mode 100644 js/src/jit/shared/CodeGenerator-shared.h create mode 100644 js/src/jit/shared/Disassembler-shared.cpp create mode 100644 js/src/jit/shared/Disassembler-shared.h create mode 100644 js/src/jit/shared/IonAssemblerBuffer.h create mode 100644 js/src/jit/shared/IonAssemblerBufferWithConstantPools.h create mode 100644 js/src/jit/shared/LIR-shared.h create mode 100644 js/src/jit/shared/Lowering-shared-inl.h create mode 100644 js/src/jit/shared/Lowering-shared.cpp create mode 100644 js/src/jit/shared/Lowering-shared.h create mode 100644 js/src/jit/wasm32/Architecture-wasm32.h create mode 100644 js/src/jit/wasm32/Assembler-wasm32.h create mode 100644 js/src/jit/wasm32/CodeGenerator-wasm32.cpp create mode 100644 js/src/jit/wasm32/CodeGenerator-wasm32.h create mode 100644 js/src/jit/wasm32/LIR-wasm32.h create mode 100644 js/src/jit/wasm32/Lowering-wasm32.h create mode 100644 js/src/jit/wasm32/MacroAssembler-wasm32-inl.h create mode 100644 js/src/jit/wasm32/MacroAssembler-wasm32.cpp create mode 100644 js/src/jit/wasm32/MacroAssembler-wasm32.h create mode 100644 js/src/jit/wasm32/MoveEmitter-wasm32.h create mode 100644 js/src/jit/wasm32/SharedICHelpers-wasm32-inl.h create mode 100644 js/src/jit/wasm32/SharedICHelpers-wasm32.h create mode 100644 js/src/jit/wasm32/SharedICRegisters-wasm32.h create mode 100644 js/src/jit/wasm32/Trampoline-wasm32.cpp create mode 100644 js/src/jit/x64/Assembler-x64.cpp create mode 100644 js/src/jit/x64/Assembler-x64.h create mode 100644 js/src/jit/x64/BaseAssembler-x64.h create mode 100644 js/src/jit/x64/CodeGenerator-x64.cpp create mode 100644 js/src/jit/x64/CodeGenerator-x64.h create mode 100644 js/src/jit/x64/LIR-x64.h create mode 100644 js/src/jit/x64/Lowering-x64.cpp create mode 100644 js/src/jit/x64/Lowering-x64.h create mode 100644 js/src/jit/x64/MacroAssembler-x64-inl.h create mode 100644 js/src/jit/x64/MacroAssembler-x64.cpp create mode 100644 js/src/jit/x64/MacroAssembler-x64.h create mode 100644 js/src/jit/x64/SharedICHelpers-x64-inl.h create mode 100644 js/src/jit/x64/SharedICHelpers-x64.h create mode 100644 js/src/jit/x64/SharedICRegisters-x64.h create mode 100644 js/src/jit/x64/Trampoline-x64.cpp create mode 100644 js/src/jit/x86-shared/Architecture-x86-shared.cpp create mode 100644 js/src/jit/x86-shared/Architecture-x86-shared.h create mode 100644 js/src/jit/x86-shared/Assembler-x86-shared.cpp create mode 100644 js/src/jit/x86-shared/Assembler-x86-shared.h create mode 100644 js/src/jit/x86-shared/AssemblerBuffer-x86-shared.cpp create mode 100644 js/src/jit/x86-shared/AssemblerBuffer-x86-shared.h create mode 100644 js/src/jit/x86-shared/BaseAssembler-x86-shared.h create mode 100644 js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp create mode 100644 js/src/jit/x86-shared/CodeGenerator-x86-shared.h create mode 100644 js/src/jit/x86-shared/Constants-x86-shared.h create mode 100644 js/src/jit/x86-shared/Encoding-x86-shared.h create mode 100644 js/src/jit/x86-shared/LIR-x86-shared.h create mode 100644 js/src/jit/x86-shared/Lowering-x86-shared.cpp create mode 100644 js/src/jit/x86-shared/Lowering-x86-shared.h create mode 100644 js/src/jit/x86-shared/MacroAssembler-x86-shared-SIMD.cpp create mode 100644 js/src/jit/x86-shared/MacroAssembler-x86-shared-inl.h create mode 100644 js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp create mode 100644 js/src/jit/x86-shared/MacroAssembler-x86-shared.h create mode 100644 js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp create mode 100644 js/src/jit/x86-shared/MoveEmitter-x86-shared.h create mode 100644 js/src/jit/x86-shared/Patching-x86-shared.h create mode 100644 js/src/jit/x86/Assembler-x86.cpp create mode 100644 js/src/jit/x86/Assembler-x86.h create mode 100644 js/src/jit/x86/BaseAssembler-x86.h create mode 100644 js/src/jit/x86/CodeGenerator-x86.cpp create mode 100644 js/src/jit/x86/CodeGenerator-x86.h create mode 100644 js/src/jit/x86/LIR-x86.h create mode 100644 js/src/jit/x86/Lowering-x86.cpp create mode 100644 js/src/jit/x86/Lowering-x86.h create mode 100644 js/src/jit/x86/MacroAssembler-x86-inl.h create mode 100644 js/src/jit/x86/MacroAssembler-x86.cpp create mode 100644 js/src/jit/x86/MacroAssembler-x86.h create mode 100644 js/src/jit/x86/SharedICHelpers-x86-inl.h create mode 100644 js/src/jit/x86/SharedICHelpers-x86.h create mode 100644 js/src/jit/x86/SharedICRegisters-x86.h create mode 100644 js/src/jit/x86/Trampoline-x86.cpp (limited to 'js/src/jit') diff --git a/js/src/jit/ABIArgGenerator.h b/js/src/jit/ABIArgGenerator.h new file mode 100644 index 0000000000..d78a21e242 --- /dev/null +++ b/js/src/jit/ABIArgGenerator.h @@ -0,0 +1,109 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_ABIArgGenerator_h +#define jit_ABIArgGenerator_h + +#include "mozilla/Assertions.h" + +#include + +#include "jit/Assembler.h" +#include "jit/IonTypes.h" +#include "jit/RegisterSets.h" +#include "wasm/WasmFrame.h" + +namespace js::jit { + +static inline MIRType ToMIRType(MIRType t) { return t; } + +static inline MIRType ToMIRType(ABIArgType argType) { + switch (argType) { + case ArgType_General: + return MIRType::Pointer; + case ArgType_Float64: + return MIRType::Double; + case ArgType_Float32: + return MIRType::Float32; + case ArgType_Int32: + return MIRType::Int32; + case ArgType_Int64: + return MIRType::Int64; + default: + break; + } + MOZ_CRASH("unexpected argType"); +} + +template +class ABIArgIterBase { + ABIArgGeneratorT gen_; + const VecT& types_; + unsigned i_; + + void settle() { + if (!done()) gen_.next(ToMIRType(types_[i_])); + } + + public: + explicit ABIArgIterBase(const VecT& types) : types_(types), i_(0) { + settle(); + } + void operator++(int) { + MOZ_ASSERT(!done()); + i_++; + settle(); + } + bool done() const { return i_ == types_.length(); } + + ABIArg* operator->() { + MOZ_ASSERT(!done()); + return &gen_.current(); + } + ABIArg& operator*() { + MOZ_ASSERT(!done()); + return gen_.current(); + } + + unsigned index() const { + MOZ_ASSERT(!done()); + return i_; + } + MIRType mirType() const { + MOZ_ASSERT(!done()); + return ToMIRType(types_[i_]); + } + uint32_t stackBytesConsumedSoFar() const { + return gen_.stackBytesConsumedSoFar(); + } +}; + +// This is not an alias because we want to allow class template argument +// deduction. +template +class ABIArgIter : public ABIArgIterBase { + public: + explicit ABIArgIter(const VecT& types) + : ABIArgIterBase(types) {} +}; + +class WasmABIArgGenerator : public ABIArgGenerator { + public: + WasmABIArgGenerator() { + increaseStackOffset(wasm::FrameWithInstances::sizeOfInstanceFields()); + } +}; + +template +class WasmABIArgIter : public ABIArgIterBase { + public: + explicit WasmABIArgIter(const VecT& types) + : ABIArgIterBase(types) {} +}; + +} // namespace js::jit + +#endif /* jit_ABIArgGenerator_h */ diff --git a/js/src/jit/ABIFunctionList-inl.h b/js/src/jit/ABIFunctionList-inl.h new file mode 100644 index 0000000000..fd0c0085ec --- /dev/null +++ b/js/src/jit/ABIFunctionList-inl.h @@ -0,0 +1,252 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_ABIFunctionList_inl_h +#define jit_ABIFunctionList_inl_h + +#include "jslibmath.h" // js::NumberMod +#include "jsmath.h" // js::ecmaPow, js::ecmaHypot, js::hypot3, js::hypot4, + // js::ecmaAtan2, js::UnaryMathFunctionType, js::powi +#include "jsnum.h" // js::StringToNumberPure, js::Int32ToStringPure, + // js::NumberToStringPure + +#include "builtin/Array.h" // js::ArrayShiftMoveElements +#include "builtin/MapObject.h" // js::MapIteratorObject::next, + // js::SetIteratorObject::next +#include "builtin/Object.h" // js::ObjectClassToString +#include "builtin/RegExp.h" // js::RegExpPrototypeOptimizableRaw, + // js::RegExpInstanceOptimizableRaw +#include "builtin/TestingFunctions.h" // js::FuzzilliHash* + +#include "irregexp/RegExpAPI.h" +// js::irregexp::CaseInsensitiveCompareNonUnicode, +// js::irregexp::CaseInsensitiveCompareUnicode, +// js::irregexp::GrowBacktrackStack, +// js::irregexp::IsCharacterInRangeArray + +#include "jit/ABIFunctions.h" +#include "jit/Bailouts.h" // js::jit::FinishBailoutToBaseline, js::jit::Bailout, + // js::jit::InvalidationBailout + +#include "jit/Ion.h" // js::jit::LazyLinkTopActivation +#include "jit/JitFrames.h" // HandleException +#include "jit/VMFunctions.h" // Rest of js::jit::* functions. + +#include "js/CallArgs.h" // JSNative +#include "js/Conversions.h" // JS::ToInt32 +// JSJitGetterOp, JSJitSetterOp, JSJitMethodOp +#include "js/experimental/JitInfo.h" +#include "js/Utility.h" // js_free + +#include "proxy/Proxy.h" // js::ProxyGetProperty + +#include "vm/ArgumentsObject.h" // js::ArgumentsObject::finishForIonPure +#include "vm/Interpreter.h" // js::TypeOfObject +#include "vm/NativeObject.h" // js::NativeObject +#include "vm/RegExpShared.h" // js::ExecuteRegExpAtomRaw +#include "wasm/WasmBuiltins.h" // js::wasm::* + +#include "builtin/Boolean-inl.h" // js::EmulatesUndefined + +namespace js { +namespace jit { + +// List of all ABI functions to be used with callWithABI. Each entry stores +// the fully qualified name of the C++ function. This list must be sorted. +#if JS_GC_PROBES +# define ABIFUNCTION_JS_GC_PROBES_LIST(_) _(js::jit::TraceCreateObject) +#else +# define ABIFUNCTION_JS_GC_PROBES_LIST(_) +#endif + +#if defined(JS_CODEGEN_ARM) +# define ABIFUNCTION_JS_CODEGEN_ARM_LIST(_) \ + _(__aeabi_idivmod) \ + _(__aeabi_uidivmod) +#else +# define ABIFUNCTION_JS_CODEGEN_ARM_LIST(_) +#endif + +#ifdef WASM_CODEGEN_DEBUG +# define ABIFUNCTION_WASM_CODEGEN_DEBUG_LIST(_) \ + _(js::wasm::PrintF32) \ + _(js::wasm::PrintF64) \ + _(js::wasm::PrintI32) \ + _(js::wasm::PrintPtr) \ + _(js::wasm::PrintText) +#else +# define ABIFUNCTION_WASM_CODEGEN_DEBUG_LIST(_) +#endif + +#ifdef FUZZING_JS_FUZZILLI +# define ABIFUNCTION_FUZZILLI_LIST(_) _(js::FuzzilliHashBigInt) +#else +# define ABIFUNCTION_FUZZILLI_LIST(_) +#endif + +#define ABIFUNCTION_LIST(_) \ + ABIFUNCTION_JS_GC_PROBES_LIST(_) \ + ABIFUNCTION_JS_CODEGEN_ARM_LIST(_) \ + ABIFUNCTION_WASM_CODEGEN_DEBUG_LIST(_) \ + _(js::ArgumentsObject::finishForIonPure) \ + _(js::ArgumentsObject::finishInlineForIonPure) \ + _(js::ArrayShiftMoveElements) \ + _(js::ecmaAtan2) \ + _(js::ecmaHypot) \ + _(js::ecmaPow) \ + _(js::EmulatesUndefined) \ + _(js::ExecuteRegExpAtomRaw) \ + _(js_free) \ + _(js::hypot3) \ + _(js::hypot4) \ + _(js::Interpret) \ + _(js::Int32ToStringPure) \ + _(js::irregexp::CaseInsensitiveCompareNonUnicode) \ + _(js::irregexp::CaseInsensitiveCompareUnicode) \ + _(js::irregexp::GrowBacktrackStack) \ + _(js::irregexp::IsCharacterInRangeArray) \ + _(js::jit::AllocateAndInitTypedArrayBuffer) \ + _(js::jit::AllocateBigIntNoGC) \ + _(js::jit::AllocateFatInlineString) \ + _(js::jit::AllocateDependentString) \ + _(js::jit::ArrayPushDensePure) \ + _(js::jit::AssertMapObjectHash) \ + _(js::jit::AssertPropertyLookup) \ + _(js::jit::AssertSetObjectHash) \ + _(js::jit::AssertValidBigIntPtr) \ + _(js::jit::AssertValidObjectPtr) \ + _(js::jit::AssertValidStringPtr) \ + _(js::jit::AssertValidSymbolPtr) \ + _(js::jit::AssertValidValue) \ + _(js::jit::AssumeUnreachable) \ + _(js::jit::AtomicsStore64) \ + _(js::jit::AtomizeStringNoGC) \ + _(js::jit::Bailout) \ + _(js::jit::BigIntNumberEqual) \ + _(js::jit::BigIntNumberEqual) \ + _(js::jit::BigIntNumberCompare) \ + _(js::jit::NumberBigIntCompare) \ + _(js::jit::NumberBigIntCompare) \ + _(js::jit::BigIntNumberCompare) \ + _(js::jit::CreateMatchResultFallbackFunc) \ + _(js::jit::EqualStringsHelperPure) \ + _(js::jit::FinishBailoutToBaseline) \ + _(js::jit::FrameIsDebuggeeCheck) \ + _(js::jit::GetContextSensitiveInterpreterStub) \ + _(js::jit::GetIndexFromString) \ + _(js::jit::GetInt32FromStringPure) \ + _(js::jit::GetNativeDataPropertyPure) \ + _(js::jit::GetNativeDataPropertyPureWithCacheLookup) \ + _(js::jit::GetNativeDataPropertyByValuePure) \ + _(js::jit::GlobalHasLiveOnDebuggerStatement) \ + _(js::jit::HandleCodeCoverageAtPC) \ + _(js::jit::HandleCodeCoverageAtPrologue) \ + _(js::jit::HandleException) \ + _(js::jit::HasNativeDataPropertyPure) \ + _(js::jit::HasNativeDataPropertyPure) \ + _(js::jit::HasNativeElementPure) \ + _(js::jit::InitBaselineFrameForOsr) \ + _(js::jit::InvalidationBailout) \ + _(js::jit::InvokeFromInterpreterStub) \ + _(js::jit::LazyLinkTopActivation) \ + _(js::jit::LinearizeForCharAccessPure) \ + _(js::jit::ObjectHasGetterSetterPure) \ + _(js::jit::ObjectIsCallable) \ + _(js::jit::ObjectIsConstructor) \ + _(js::jit::PostGlobalWriteBarrier) \ + _(js::jit::PostWriteBarrier) \ + _(js::jit::PostWriteElementBarrier) \ + _(js::jit::PostWriteElementBarrier) \ + _(js::jit::Printf0) \ + _(js::jit::Printf1) \ + _(js::jit::StringFromCharCodeNoGC) \ + _(js::jit::TypeOfNameObject) \ + _(js::jit::WrapObjectPure) \ + ABIFUNCTION_FUZZILLI_LIST(_) \ + _(js::MapIteratorObject::next) \ + _(js::NativeObject::addDenseElementPure) \ + _(js::NativeObject::growSlotsPure) \ + _(js::NumberMod) \ + _(js::NumberToStringPure) \ + _(js::ObjectClassToString) \ + _(js::powi) \ + _(js::ProxyGetProperty) \ + _(js::RegExpInstanceOptimizableRaw) \ + _(js::RegExpPrototypeOptimizableRaw) \ + _(js::SetIteratorObject::next) \ + _(js::StringToNumberPure) \ + _(js::TypeOfObject) + +// List of all ABI functions to be used with callWithABI, which are +// overloaded. Each entry stores the fully qualified name of the C++ function, +// followed by the signature of the function to be called. When the function +// is not overloaded, you should prefer adding the function to +// ABIFUNCTION_LIST instead. This list must be sorted with the name of the C++ +// function. +#define ABIFUNCTION_AND_TYPE_LIST(_) _(JS::ToInt32, int32_t (*)(double)) + +// List of all ABI function signature which are using a computed function +// pointer instead of a statically known function pointer. +#define ABIFUNCTIONSIG_LIST(_) \ + _(AtomicsCompareExchangeFn) \ + _(AtomicsReadWriteModifyFn) \ + _(bool (*)(BigInt*, BigInt*)) \ + _(bool (*)(BigInt*, double)) \ + _(bool (*)(double, BigInt*)) \ + _(float (*)(float)) \ + _(JSJitGetterOp) \ + _(JSJitMethodOp) \ + _(JSJitSetterOp) \ + _(JSNative) \ + _(js::UnaryMathFunctionType) \ + _(void (*)(js::gc::StoreBuffer*, js::gc::Cell**)) \ + _(void (*)(JSRuntime * rt, JSObject * *objp)) \ + _(void (*)(JSRuntime * rt, JSString * *stringp)) \ + _(void (*)(JSRuntime * rt, Shape * *shapep)) \ + _(void (*)(JSRuntime * rt, Value * vp)) + +// GCC warns when the signature does not have matching attributes (for example +// [[nodiscard]]). Squelch this warning to avoid a GCC-only footgun. +#if MOZ_IS_GCC +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wignored-attributes" +#endif + +// Note: the use of ::fp instead of fp is intentional to enforce use of +// fully-qualified names in the list above. +#define DEF_TEMPLATE(fp) \ + template <> \ + struct ABIFunctionData { \ + static constexpr bool registered = true; \ + }; +ABIFUNCTION_LIST(DEF_TEMPLATE) +#undef DEF_TEMPLATE + +#define DEF_TEMPLATE(fp, ...) \ + template <> \ + struct ABIFunctionData<__VA_ARGS__, ::fp> { \ + static constexpr bool registered = true; \ + }; +ABIFUNCTION_AND_TYPE_LIST(DEF_TEMPLATE) +#undef DEF_TEMPLATE + +// Define a known list of function signatures. +#define DEF_TEMPLATE(...) \ + template <> \ + struct ABIFunctionSignatureData<__VA_ARGS__> { \ + static constexpr bool registered = true; \ + }; +ABIFUNCTIONSIG_LIST(DEF_TEMPLATE) +#undef DEF_TEMPLATE + +#if MOZ_IS_GCC +# pragma GCC diagnostic pop +#endif + +} // namespace jit +} // namespace js + +#endif // jit_VMFunctionList_inl_h diff --git a/js/src/jit/ABIFunctions.h b/js/src/jit/ABIFunctions.h new file mode 100644 index 0000000000..d6bd15555f --- /dev/null +++ b/js/src/jit/ABIFunctions.h @@ -0,0 +1,73 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_ABIFunctions_h +#define jit_ABIFunctions_h + +#include "jstypes.h" // JS_FUNC_TO_DATA_PTR + +namespace js { +namespace jit { + +// This class is used to ensure that all known targets of callWithABI are +// registered here. Otherwise, this would raise a static assertion at compile +// time. +template +struct ABIFunctionData { + static const bool registered = false; +}; + +template +struct ABIFunction { + void* address() const { return JS_FUNC_TO_DATA_PTR(void*, fun); } + + // If this assertion fails, you are likely in the context of a + // `callWithABI()` call. This error indicates that ABIFunction has + // not been specialized for `` by the time of this call. + // + // This can be fixed by adding the function signature to either + // ABIFUNCTION_LIST or ABIFUNCTION_AND_TYPE_LIST (if overloaded) within + // `ABIFunctionList-inl.h` and to add an `#include` statement of this header + // in the file which is making the call to `callWithABI()`. + static_assert(ABIFunctionData::registered, + "ABI function is not registered."); +}; + +template +struct ABIFunctionSignatureData { + static const bool registered = false; +}; + +template +struct ABIFunctionSignature { + void* address(Sig fun) const { return JS_FUNC_TO_DATA_PTR(void*, fun); } + + // If this assertion fails, you are likely in the context of a + // `DynamicFunction(fn)` call. This error indicates that + // ABIFunctionSignature has not been specialized for `Sig` by the time of this + // call. + // + // This can be fixed by adding the function signature to ABIFUNCTIONSIG_LIST + // within `ABIFunctionList-inl.h` and to add an `#include` statement of this + // header in the file which is making the call to `DynamicFunction(fn)`. + static_assert(ABIFunctionSignatureData::registered, + "ABI function signature is not registered."); +}; + +// This is a structure created to ensure that the dynamically computed +// function pointer is well typed. +// +// It is meant to be created only through DynamicFunction function calls. In +// extremelly rare cases, such as VMFunctions, it might be produced as a result +// of GetVMFunctionTarget. +struct DynFn { + void* address; +}; + +} // namespace jit +} // namespace js + +#endif /* jit_VMFunctions_h */ diff --git a/js/src/jit/AliasAnalysis.cpp b/js/src/jit/AliasAnalysis.cpp new file mode 100644 index 0000000000..8334d55dfe --- /dev/null +++ b/js/src/jit/AliasAnalysis.cpp @@ -0,0 +1,317 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/AliasAnalysis.h" + +#include "jit/JitSpewer.h" +#include "jit/MIR.h" +#include "jit/MIRGenerator.h" +#include "jit/MIRGraph.h" + +#include "js/Printer.h" + +using namespace js; +using namespace js::jit; + +namespace js { +namespace jit { + +class LoopAliasInfo : public TempObject { + private: + LoopAliasInfo* outer_; + MBasicBlock* loopHeader_; + MInstructionVector invariantLoads_; + + public: + LoopAliasInfo(TempAllocator& alloc, LoopAliasInfo* outer, + MBasicBlock* loopHeader) + : outer_(outer), loopHeader_(loopHeader), invariantLoads_(alloc) {} + + MBasicBlock* loopHeader() const { return loopHeader_; } + LoopAliasInfo* outer() const { return outer_; } + bool addInvariantLoad(MInstruction* ins) { + return invariantLoads_.append(ins); + } + const MInstructionVector& invariantLoads() const { return invariantLoads_; } + MInstruction* firstInstruction() const { return *loopHeader_->begin(); } +}; + +} // namespace jit +} // namespace js + +void AliasAnalysis::spewDependencyList() { +#ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_AliasSummaries)) { + Fprinter& print = JitSpewPrinter(); + JitSpewHeader(JitSpew_AliasSummaries); + print.printf("Dependency list for other passes:\n"); + + for (ReversePostorderIterator block(graph_.rpoBegin()); + block != graph_.rpoEnd(); block++) { + for (MInstructionIterator def(block->begin()), + end(block->begin(block->lastIns())); + def != end; ++def) { + if (!def->dependency()) { + continue; + } + if (!def->getAliasSet().isLoad()) { + continue; + } + + JitSpewHeader(JitSpew_AliasSummaries); + print.printf(" "); + MDefinition::PrintOpcodeName(print, def->op()); + print.printf("%u marked depending on ", def->id()); + MDefinition::PrintOpcodeName(print, def->dependency()->op()); + print.printf("%u\n", def->dependency()->id()); + } + } + } +#endif +} + +// Whether there might be a path from src to dest, excluding loop backedges. +// This is approximate and really ought to depend on precomputed reachability +// information. +static inline bool BlockMightReach(MBasicBlock* src, MBasicBlock* dest) { + while (src->id() <= dest->id()) { + if (src == dest) { + return true; + } + switch (src->numSuccessors()) { + case 0: + return false; + case 1: { + MBasicBlock* successor = src->getSuccessor(0); + if (successor->id() <= src->id()) { + return true; // Don't iloop. + } + src = successor; + break; + } + default: + return true; + } + } + return false; +} + +static void IonSpewDependency(MInstruction* load, MInstruction* store, + const char* verb, const char* reason) { +#ifdef JS_JITSPEW + if (!JitSpewEnabled(JitSpew_Alias)) { + return; + } + + JitSpewHeader(JitSpew_Alias); + Fprinter& out = JitSpewPrinter(); + out.printf(" Load "); + load->printName(out); + out.printf(" %s on store ", verb); + store->printName(out); + out.printf(" (%s)\n", reason); +#endif +} + +static void IonSpewAliasInfo(const char* pre, MInstruction* ins, + const char* post) { +#ifdef JS_JITSPEW + if (!JitSpewEnabled(JitSpew_Alias)) { + return; + } + + JitSpewHeader(JitSpew_Alias); + Fprinter& out = JitSpewPrinter(); + out.printf(" %s ", pre); + ins->printName(out); + out.printf(" %s\n", post); +#endif +} + +// [SMDOC] IonMonkey Alias Analysis +// +// This pass annotates every load instruction with the last store instruction +// on which it depends. The algorithm is optimistic in that it ignores explicit +// dependencies and only considers loads and stores. +// +// Loads inside loops only have an implicit dependency on a store before the +// loop header if no instruction inside the loop body aliases it. To calculate +// this efficiently, we maintain a list of maybe-invariant loads and the +// combined alias set for all stores inside the loop. When we see the loop's +// backedge, this information is used to mark every load we wrongly assumed to +// be loop invariant as having an implicit dependency on the last instruction of +// the loop header, so that it's never moved before the loop header. +// +// The algorithm depends on the invariant that both control instructions and +// effectful instructions (stores) are never hoisted. +bool AliasAnalysis::analyze() { + JitSpew(JitSpew_Alias, "Begin"); + Vector stores( + alloc()); + + // Initialize to the first instruction. + MInstruction* firstIns = *graph_.entryBlock()->begin(); + for (unsigned i = 0; i < AliasSet::NumCategories; i++) { + MInstructionVector defs(alloc()); + if (!defs.append(firstIns)) { + return false; + } + if (!stores.append(std::move(defs))) { + return false; + } + } + + // Type analysis may have inserted new instructions. Since this pass depends + // on the instruction number ordering, all instructions are renumbered. + uint32_t newId = 0; + + for (ReversePostorderIterator block(graph_.rpoBegin()); + block != graph_.rpoEnd(); block++) { + if (mir->shouldCancel("Alias Analysis (main loop)")) { + return false; + } + + if (block->isLoopHeader()) { + JitSpew(JitSpew_Alias, "Processing loop header %u", block->id()); + loop_ = new (alloc().fallible()) LoopAliasInfo(alloc(), loop_, *block); + if (!loop_) { + return false; + } + } + + for (MPhiIterator def(block->phisBegin()), end(block->phisEnd()); + def != end; ++def) { + def->setId(newId++); + } + + for (MInstructionIterator def(block->begin()), end(block->end()); + def != end; ++def) { + def->setId(newId++); + + AliasSet set = def->getAliasSet(); + if (set.isNone()) { + continue; + } + + // For the purposes of alias analysis, all recoverable operations + // are treated as effect free as the memory represented by these + // operations cannot be aliased by others. + if (def->canRecoverOnBailout()) { + continue; + } + + if (set.isStore()) { + for (AliasSetIterator iter(set); iter; iter++) { + if (!stores[*iter].append(*def)) { + return false; + } + } + +#ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_Alias)) { + JitSpewHeader(JitSpew_Alias); + Fprinter& out = JitSpewPrinter(); + out.printf("Processing store "); + def->printName(out); + out.printf(" (flags %x)\n", set.flags()); + } +#endif + } else { + // Find the most recent store on which this instruction depends. + MInstruction* lastStore = firstIns; + + for (AliasSetIterator iter(set); iter; iter++) { + MInstructionVector& aliasedStores = stores[*iter]; + for (int i = aliasedStores.length() - 1; i >= 0; i--) { + MInstruction* store = aliasedStores[i]; + if (def->mightAlias(store) != MDefinition::AliasType::NoAlias && + BlockMightReach(store->block(), *block)) { + if (lastStore->id() < store->id()) { + lastStore = store; + } + break; + } + } + } + + def->setDependency(lastStore); + IonSpewDependency(*def, lastStore, "depends", ""); + + // If the last store was before the current loop, we assume this load + // is loop invariant. If a later instruction writes to the same + // location, we will fix this at the end of the loop. + if (loop_ && lastStore->id() < loop_->firstInstruction()->id()) { + if (!loop_->addInvariantLoad(*def)) { + return false; + } + } + } + } + + if (block->isLoopBackedge()) { + MOZ_ASSERT(loop_->loopHeader() == block->loopHeaderOfBackedge()); + JitSpew(JitSpew_Alias, "Processing loop backedge %u (header %u)", + block->id(), loop_->loopHeader()->id()); + LoopAliasInfo* outerLoop = loop_->outer(); + MInstruction* firstLoopIns = *loop_->loopHeader()->begin(); + + const MInstructionVector& invariant = loop_->invariantLoads(); + + for (unsigned i = 0; i < invariant.length(); i++) { + MInstruction* ins = invariant[i]; + AliasSet set = ins->getAliasSet(); + MOZ_ASSERT(set.isLoad()); + + bool hasAlias = false; + for (AliasSetIterator iter(set); iter; iter++) { + MInstructionVector& aliasedStores = stores[*iter]; + for (int i = aliasedStores.length() - 1;; i--) { + MInstruction* store = aliasedStores[i]; + if (store->id() < firstLoopIns->id()) { + break; + } + if (ins->mightAlias(store) != MDefinition::AliasType::NoAlias) { + hasAlias = true; + IonSpewDependency(ins, store, "aliases", "store in loop body"); + break; + } + } + if (hasAlias) { + break; + } + } + + if (hasAlias) { + // This instruction depends on stores inside the loop body. Mark it as + // having a dependency on the last instruction of the loop header. The + // last instruction is a control instruction and these are never + // hoisted. + MControlInstruction* controlIns = loop_->loopHeader()->lastIns(); + IonSpewDependency(ins, controlIns, "depends", + "due to stores in loop body"); + ins->setDependency(controlIns); + } else { + IonSpewAliasInfo("Load", ins, + "does not depend on any stores in this loop"); + + if (outerLoop && + ins->dependency()->id() < outerLoop->firstInstruction()->id()) { + IonSpewAliasInfo("Load", ins, "may be invariant in outer loop"); + if (!outerLoop->addInvariantLoad(ins)) { + return false; + } + } + } + } + loop_ = loop_->outer(); + } + } + + spewDependencyList(); + + MOZ_ASSERT(loop_ == nullptr); + return true; +} diff --git a/js/src/jit/AliasAnalysis.h b/js/src/jit/AliasAnalysis.h new file mode 100644 index 0000000000..49ddaee47c --- /dev/null +++ b/js/src/jit/AliasAnalysis.h @@ -0,0 +1,64 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_AliasAnalysis_h +#define jit_AliasAnalysis_h + +#include "jit/MIR.h" +#include "jit/MIRGraph.h" + +namespace js { +namespace jit { + +class LoopAliasInfo; + +class AliasAnalysis { + MIRGenerator* mir; + MIRGraph& graph_; + LoopAliasInfo* loop_; + + void spewDependencyList(); + + TempAllocator& alloc() const { return graph_.alloc(); } + + public: + AliasAnalysis(MIRGenerator* mir, MIRGraph& graph) + : mir(mir), graph_(graph), loop_(nullptr) {} + + [[nodiscard]] bool analyze(); +}; + +// Iterates over the flags in an AliasSet. +class AliasSetIterator { + private: + uint32_t flags; + unsigned pos; + + public: + explicit AliasSetIterator(AliasSet set) : flags(set.flags()), pos(0) { + while (flags && (flags & 1) == 0) { + flags >>= 1; + pos++; + } + } + AliasSetIterator& operator++(int) { + do { + flags >>= 1; + pos++; + } while (flags && (flags & 1) == 0); + return *this; + } + explicit operator bool() const { return !!flags; } + unsigned operator*() const { + MOZ_ASSERT(pos < AliasSet::NumCategories); + return pos; + } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_AliasAnalysis_h */ diff --git a/js/src/jit/AlignmentMaskAnalysis.cpp b/js/src/jit/AlignmentMaskAnalysis.cpp new file mode 100644 index 0000000000..5b19b0861c --- /dev/null +++ b/js/src/jit/AlignmentMaskAnalysis.cpp @@ -0,0 +1,97 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/AlignmentMaskAnalysis.h" +#include "jit/MIR.h" +#include "jit/MIRGraph.h" + +using namespace js; +using namespace jit; + +static bool IsAlignmentMask(uint32_t m) { + // Test whether m is just leading ones and trailing zeros. + return (-m & ~m) == 0; +} + +static void AnalyzeAsmHeapAddress(MDefinition* ptr, MIRGraph& graph) { + // Fold (a+i)&m to (a&m)+i, provided that this doesn't change the result, + // since the users of the BitAnd include heap accesses. This will expose + // the redundancy for GVN when expressions like this: + // a&m + // (a+1)&m, + // (a+2)&m, + // are transformed into this: + // a&m + // (a&m)+1 + // (a&m)+2 + // and it will allow the constants to be folded by the + // EffectiveAddressAnalysis pass. + // + // Putting the add on the outside might seem like it exposes other users of + // the expression to the possibility of i32 overflow, if we aren't in wasm + // and they aren't naturally truncating. However, since we use MAdd::New + // with MIRType::Int32, we make sure that the value is truncated, just as it + // would be by the MBitAnd. + + MOZ_ASSERT(IsCompilingWasm()); + + if (!ptr->isBitAnd()) { + return; + } + + MDefinition* lhs = ptr->toBitAnd()->getOperand(0); + MDefinition* rhs = ptr->toBitAnd()->getOperand(1); + if (lhs->isConstant()) { + std::swap(lhs, rhs); + } + if (!lhs->isAdd() || !rhs->isConstant()) { + return; + } + + MDefinition* op0 = lhs->toAdd()->getOperand(0); + MDefinition* op1 = lhs->toAdd()->getOperand(1); + if (op0->isConstant()) { + std::swap(op0, op1); + } + if (!op1->isConstant()) { + return; + } + + uint32_t i = op1->toConstant()->toInt32(); + uint32_t m = rhs->toConstant()->toInt32(); + if (!IsAlignmentMask(m) || (i & m) != i) { + return; + } + + // The pattern was matched! Produce the replacement expression. + MInstruction* and_ = MBitAnd::New(graph.alloc(), op0, rhs, MIRType::Int32); + ptr->block()->insertBefore(ptr->toBitAnd(), and_); + auto* add = MAdd::New(graph.alloc(), and_, op1, TruncateKind::Truncate); + ptr->block()->insertBefore(ptr->toBitAnd(), add); + ptr->replaceAllUsesWith(add); + ptr->block()->discard(ptr->toBitAnd()); +} + +bool AlignmentMaskAnalysis::analyze() { + for (ReversePostorderIterator block(graph_.rpoBegin()); + block != graph_.rpoEnd(); block++) { + for (MInstructionIterator i = block->begin(); i != block->end(); i++) { + if (!graph_.alloc().ensureBallast()) { + return false; + } + + // Note that we don't check for MWasmCompareExchangeHeap + // or MWasmAtomicBinopHeap, because the backend and the OOB + // mechanism don't support non-zero offsets for them yet. + if (i->isAsmJSLoadHeap()) { + AnalyzeAsmHeapAddress(i->toAsmJSLoadHeap()->base(), graph_); + } else if (i->isAsmJSStoreHeap()) { + AnalyzeAsmHeapAddress(i->toAsmJSStoreHeap()->base(), graph_); + } + } + } + return true; +} diff --git a/js/src/jit/AlignmentMaskAnalysis.h b/js/src/jit/AlignmentMaskAnalysis.h new file mode 100644 index 0000000000..4e46ca97d3 --- /dev/null +++ b/js/src/jit/AlignmentMaskAnalysis.h @@ -0,0 +1,27 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_AlignmentMaskAnalysis_h +#define jit_AlignmentMaskAnalysis_h + +namespace js { +namespace jit { + +class MIRGraph; + +class AlignmentMaskAnalysis { + MIRGraph& graph_; + + public: + explicit AlignmentMaskAnalysis(MIRGraph& graph) : graph_(graph) {} + + [[nodiscard]] bool analyze(); +}; + +} /* namespace jit */ +} /* namespace js */ + +#endif /* jit_AlignmentMaskAnalysis_h */ diff --git a/js/src/jit/Assembler.h b/js/src/jit/Assembler.h new file mode 100644 index 0000000000..5003c351ac --- /dev/null +++ b/js/src/jit/Assembler.h @@ -0,0 +1,34 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_Assembler_h +#define jit_Assembler_h + +#if defined(JS_CODEGEN_X86) +# include "jit/x86/Assembler-x86.h" +#elif defined(JS_CODEGEN_X64) +# include "jit/x64/Assembler-x64.h" +#elif defined(JS_CODEGEN_ARM) +# include "jit/arm/Assembler-arm.h" +#elif defined(JS_CODEGEN_ARM64) +# include "jit/arm64/Assembler-arm64.h" +#elif defined(JS_CODEGEN_MIPS32) +# include "jit/mips32/Assembler-mips32.h" +#elif defined(JS_CODEGEN_MIPS64) +# include "jit/mips64/Assembler-mips64.h" +#elif defined(JS_CODEGEN_LOONG64) +# include "jit/loong64/Assembler-loong64.h" +#elif defined(JS_CODEGEN_RISCV64) +# include "jit/riscv64/Assembler-riscv64.h" +#elif defined(JS_CODEGEN_WASM32) +# include "jit/wasm32/Assembler-wasm32.h" +#elif defined(JS_CODEGEN_NONE) +# include "jit/none/Assembler-none.h" +#else +# error "Unknown architecture!" +#endif + +#endif /* jit_Assembler_h */ diff --git a/js/src/jit/AtomicOp.h b/js/src/jit/AtomicOp.h new file mode 100644 index 0000000000..90edb631cb --- /dev/null +++ b/js/src/jit/AtomicOp.h @@ -0,0 +1,98 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_AtomicOp_h +#define jit_AtomicOp_h + +namespace js { +namespace jit { + +// Types of atomic operation, shared by MIR and LIR. + +enum AtomicOp { + AtomicFetchAddOp, + AtomicFetchSubOp, + AtomicFetchAndOp, + AtomicFetchOrOp, + AtomicFetchXorOp +}; + +// Memory barrier types, shared by MIR and LIR. +// +// MembarSynchronizing is here because some platforms can make the +// distinction (DSB vs DMB on ARM, SYNC vs parameterized SYNC on MIPS) +// but there's been no reason to use it yet. + +enum MemoryBarrierBits { + MembarLoadLoad = 1, + MembarLoadStore = 2, + MembarStoreStore = 4, + MembarStoreLoad = 8, + + MembarSynchronizing = 16, + + // For validity testing + MembarNobits = 0, + MembarAllbits = 31, +}; + +static inline constexpr MemoryBarrierBits operator|(MemoryBarrierBits a, + MemoryBarrierBits b) { + return MemoryBarrierBits(int(a) | int(b)); +} + +static inline constexpr MemoryBarrierBits operator&(MemoryBarrierBits a, + MemoryBarrierBits b) { + return MemoryBarrierBits(int(a) & int(b)); +} + +static inline constexpr MemoryBarrierBits operator~(MemoryBarrierBits a) { + return MemoryBarrierBits(~int(a)); +} + +// Standard barrier bits for a full barrier. +static constexpr MemoryBarrierBits MembarFull = + MembarLoadLoad | MembarLoadStore | MembarStoreLoad | MembarStoreStore; + +// Standard sets of barrier bits for atomic loads and stores. +// See http://gee.cs.oswego.edu/dl/jmm/cookbook.html for more. +static constexpr MemoryBarrierBits MembarBeforeLoad = MembarNobits; +static constexpr MemoryBarrierBits MembarAfterLoad = + MembarLoadLoad | MembarLoadStore; +static constexpr MemoryBarrierBits MembarBeforeStore = MembarStoreStore; +static constexpr MemoryBarrierBits MembarAfterStore = MembarStoreLoad; + +struct Synchronization { + const MemoryBarrierBits barrierBefore; + const MemoryBarrierBits barrierAfter; + + constexpr Synchronization(MemoryBarrierBits before, MemoryBarrierBits after) + : barrierBefore(before), barrierAfter(after) {} + + static Synchronization None() { + return Synchronization(MemoryBarrierBits(MembarNobits), + MemoryBarrierBits(MembarNobits)); + } + + static Synchronization Full() { + return Synchronization(MembarFull, MembarFull); + } + + static Synchronization Load() { + return Synchronization(MembarBeforeLoad, MembarAfterLoad); + } + + static Synchronization Store() { + return Synchronization(MembarBeforeStore, MembarAfterStore); + } + + bool isNone() const { return (barrierBefore | barrierAfter) == MembarNobits; } +}; + +} // namespace jit +} // namespace js + +#endif /* jit_AtomicOp_h */ diff --git a/js/src/jit/AtomicOperations.h b/js/src/jit/AtomicOperations.h new file mode 100644 index 0000000000..8ad2839b36 --- /dev/null +++ b/js/src/jit/AtomicOperations.h @@ -0,0 +1,352 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_AtomicOperations_h +#define jit_AtomicOperations_h + +#include "mozilla/Types.h" + +#include + +#include "jit/AtomicOperationsGenerated.h" +#include "vm/SharedMem.h" + +namespace js { +namespace jit { + +/* + * [SMDOC] Atomic Operations + * + * The atomic operations layer defines types and functions for + * JIT-compatible atomic operation. + * + * The fundamental constraints on the functions are: + * + * - That their realization here MUST be compatible with code the JIT + * generates for its Atomics operations, so that an atomic access + * from the interpreter or runtime - from any C++ code - really is + * atomic relative to a concurrent, compatible atomic access from + * jitted code. That is, these primitives expose JIT-compatible + * atomicity functionality to C++. + * + * - That accesses may race without creating C++ undefined behavior: + * atomic accesses (marked "SeqCst") may race with non-atomic + * accesses (marked "SafeWhenRacy"); overlapping but non-matching, + * and hence incompatible, atomic accesses may race; and non-atomic + * accesses may race. The effects of races need not be predictable, + * so garbage can be produced by a read or written by a write, but + * the effects must be benign: the program must continue to run, and + * only the memory in the union of addresses named in the racing + * accesses may be affected. + * + * The compatibility constraint means that if the JIT makes dynamic + * decisions about how to implement atomic operations then + * corresponding dynamic decisions MUST be made in the implementations + * of the functions below. + * + * The safe-for-races constraint means that by and large, it is hard + * to implement these primitives in C++. See "Implementation notes" + * below. + * + * The "SeqCst" suffix on operations means "sequentially consistent" + * and means such a function's operation must have "sequentially + * consistent" memory ordering. See mfbt/Atomics.h for an explanation + * of this memory ordering. + * + * Note that a "SafeWhenRacy" access does not provide the atomicity of + * a "relaxed atomic" access: it can read or write garbage if there's + * a race. + * + * + * Implementation notes. + * + * It's not a requirement that these functions be inlined; performance + * is not a great concern. On some platforms these functions may call + * functions that use inline assembly. See GenerateAtomicOperations.py. + * + * In principle these functions will not be written in C++, thus + * making races defined behavior if all racy accesses from C++ go via + * these functions. (Jitted code will always be safe for races and + * provides the same guarantees as these functions.) + * + * The appropriate implementations will be platform-specific and + * there are some obvious implementation strategies to choose + * from, sometimes a combination is appropriate: + * + * - generating the code at run-time with the JIT; + * - hand-written assembler (maybe inline); or + * - using special compiler intrinsics or directives. + * + * Trusting the compiler not to generate code that blows up on a + * race definitely won't work in the presence of TSan, or even of + * optimizing compilers in seemingly-"innocuous" conditions. (See + * https://www.usenix.org/legacy/event/hotpar11/tech/final_files/Boehm.pdf + * for details.) + */ +class AtomicOperations { + // The following functions are defined for T = int8_t, uint8_t, + // int16_t, uint16_t, int32_t, uint32_t, int64_t, and uint64_t. + + // Atomically read *addr. + template + static inline T loadSeqCst(T* addr); + + // Atomically store val in *addr. + template + static inline void storeSeqCst(T* addr, T val); + + // Atomically store val in *addr and return the old value of *addr. + template + static inline T exchangeSeqCst(T* addr, T val); + + // Atomically check that *addr contains oldval and if so replace it + // with newval, in any case returning the old contents of *addr. + template + static inline T compareExchangeSeqCst(T* addr, T oldval, T newval); + + // Atomically add, subtract, bitwise-AND, bitwise-OR, or bitwise-XOR + // val into *addr and return the old value of *addr. + template + static inline T fetchAddSeqCst(T* addr, T val); + + template + static inline T fetchSubSeqCst(T* addr, T val); + + template + static inline T fetchAndSeqCst(T* addr, T val); + + template + static inline T fetchOrSeqCst(T* addr, T val); + + template + static inline T fetchXorSeqCst(T* addr, T val); + + // The SafeWhenRacy functions are to be used when C++ code has to access + // memory without synchronization and can't guarantee that there won't be a + // race on the access. But they are access-atomic for integer data so long + // as any racing writes are of the same size and to the same address. + + // Defined for all the integral types as well as for float32 and float64, + // but not access-atomic for floats, nor for int64 and uint64 on 32-bit + // platforms. + template + static inline T loadSafeWhenRacy(T* addr); + + // Defined for all the integral types as well as for float32 and float64, + // but not access-atomic for floats, nor for int64 and uint64 on 32-bit + // platforms. + template + static inline void storeSafeWhenRacy(T* addr, T val); + + // Replacement for memcpy(). No access-atomicity guarantees. + static inline void memcpySafeWhenRacy(void* dest, const void* src, + size_t nbytes); + + // Replacement for memmove(). No access-atomicity guarantees. + static inline void memmoveSafeWhenRacy(void* dest, const void* src, + size_t nbytes); + + public: + // Test lock-freedom for any int32 value. This implements the + // Atomics::isLockFree() operation in the ECMAScript Shared Memory and + // Atomics specification, as follows: + // + // 4-byte accesses are always lock free (in the spec). + // 1-, 2-, and 8-byte accesses are always lock free (in SpiderMonkey). + // + // There is no lock-freedom for JS for any other values on any platform. + static constexpr inline bool isLockfreeJS(int32_t n); + + // If the return value is true then the templated functions below are + // supported for int64_t and uint64_t. If the return value is false then + // those functions will MOZ_CRASH. The value of this call does not change + // during execution. + static inline bool hasAtomic8(); + + // If the return value is true then hasAtomic8() is true and the atomic + // operations are indeed lock-free. The value of this call does not change + // during execution. + static inline bool isLockfree8(); + + // Execute a full memory barrier (LoadLoad+LoadStore+StoreLoad+StoreStore). + static inline void fenceSeqCst(); + + // All clients should use the APIs that take SharedMem pointers. + // See above for semantics and acceptable types. + + template + static T loadSeqCst(SharedMem addr) { + return loadSeqCst(addr.unwrap()); + } + + template + static void storeSeqCst(SharedMem addr, T val) { + return storeSeqCst(addr.unwrap(), val); + } + + template + static T exchangeSeqCst(SharedMem addr, T val) { + return exchangeSeqCst(addr.unwrap(), val); + } + + template + static T compareExchangeSeqCst(SharedMem addr, T oldval, T newval) { + return compareExchangeSeqCst(addr.unwrap(), oldval, newval); + } + + template + static T fetchAddSeqCst(SharedMem addr, T val) { + return fetchAddSeqCst(addr.unwrap(), val); + } + + template + static T fetchSubSeqCst(SharedMem addr, T val) { + return fetchSubSeqCst(addr.unwrap(), val); + } + + template + static T fetchAndSeqCst(SharedMem addr, T val) { + return fetchAndSeqCst(addr.unwrap(), val); + } + + template + static T fetchOrSeqCst(SharedMem addr, T val) { + return fetchOrSeqCst(addr.unwrap(), val); + } + + template + static T fetchXorSeqCst(SharedMem addr, T val) { + return fetchXorSeqCst(addr.unwrap(), val); + } + + template + static T loadSafeWhenRacy(SharedMem addr) { + return loadSafeWhenRacy(addr.unwrap()); + } + + template + static void storeSafeWhenRacy(SharedMem addr, T val) { + return storeSafeWhenRacy(addr.unwrap(), val); + } + + template + static void memcpySafeWhenRacy(SharedMem dest, SharedMem src, + size_t nbytes) { + memcpySafeWhenRacy(dest.template cast().unwrap(), + src.template cast().unwrap(), nbytes); + } + + template + static void memcpySafeWhenRacy(SharedMem dest, T* src, size_t nbytes) { + memcpySafeWhenRacy(dest.template cast().unwrap(), + static_cast(src), nbytes); + } + + template + static void memcpySafeWhenRacy(T* dest, SharedMem src, size_t nbytes) { + memcpySafeWhenRacy(static_cast(dest), + src.template cast().unwrap(), nbytes); + } + + template + static void memmoveSafeWhenRacy(SharedMem dest, SharedMem src, + size_t nbytes) { + memmoveSafeWhenRacy(dest.template cast().unwrap(), + src.template cast().unwrap(), nbytes); + } + + static void memsetSafeWhenRacy(SharedMem dest, int value, + size_t nbytes) { + uint8_t buf[1024]; + size_t iterations = nbytes / sizeof(buf); + size_t tail = nbytes % sizeof(buf); + size_t offs = 0; + if (iterations > 0) { + memset(buf, value, sizeof(buf)); + while (iterations--) { + memcpySafeWhenRacy(dest + offs, SharedMem::unshared(buf), + sizeof(buf)); + offs += sizeof(buf); + } + } else { + memset(buf, value, tail); + } + memcpySafeWhenRacy(dest + offs, SharedMem::unshared(buf), tail); + } + + template + static void podCopySafeWhenRacy(SharedMem dest, SharedMem src, + size_t nelem) { + memcpySafeWhenRacy(dest, src, nelem * sizeof(T)); + } + + template + static void podMoveSafeWhenRacy(SharedMem dest, SharedMem src, + size_t nelem) { + memmoveSafeWhenRacy(dest, src, nelem * sizeof(T)); + } +}; + +constexpr inline bool AtomicOperations::isLockfreeJS(int32_t size) { + // Keep this in sync with atomicIsLockFreeJS() in jit/MacroAssembler.cpp. + + switch (size) { + case 1: + return true; + case 2: + return true; + case 4: + // The spec requires Atomics.isLockFree(4) to return true. + return true; + case 8: + return true; + default: + return false; + } +} + +} // namespace jit +} // namespace js + +// As explained above, our atomic operations are not portable even in principle, +// so we must include platform+compiler specific definitions here. +// +// x86, x64, arm, and arm64 are maintained by Mozilla. All other platform +// setups are by platform maintainers' request and are not maintained by +// Mozilla. +// +// If you are using a platform+compiler combination that causes an error below +// (and if the problem isn't just that the compiler uses a different name for a +// known architecture), you have basically three options: +// +// - find an already-supported compiler for the platform and use that instead +// +// - write your own support code for the platform+compiler and create a new +// case below +// +// - include jit/shared/AtomicOperations-feeling-lucky.h in a case for the +// platform below, if you have a gcc-compatible compiler and truly feel +// lucky. You may have to add a little code to that file, too. +// +// Simulators are confusing. These atomic primitives must be compatible with +// the code that the JIT emits, but of course for an ARM simulator running on +// x86 the primitives here will be for x86, not for ARM, while the JIT emits ARM +// code. Our ARM simulator solves that the easy way: by using these primitives +// to implement its atomic operations. For other simulators there may need to +// be special cases below to provide simulator-compatible primitives, for +// example, for our ARM64 simulator the primitives could in principle +// participate in the memory exclusivity monitors implemented by the simulator. +// Such a solution is likely to be difficult. + +#ifdef JS_HAVE_GENERATED_ATOMIC_OPS +# include "jit/shared/AtomicOperations-shared-jit.h" +#elif defined(JS_SIMULATOR_MIPS32) || defined(__mips__) +# include "jit/mips-shared/AtomicOperations-mips-shared.h" +#else +# include "jit/shared/AtomicOperations-feeling-lucky.h" +#endif + +#endif // jit_AtomicOperations_h diff --git a/js/src/jit/AutoWritableJitCode.h b/js/src/jit/AutoWritableJitCode.h new file mode 100644 index 0000000000..ab5b35a54f --- /dev/null +++ b/js/src/jit/AutoWritableJitCode.h @@ -0,0 +1,88 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_AutoWritableJitCode_h +#define jit_AutoWritableJitCode_h + +#include "mozilla/Assertions.h" +#include "mozilla/Attributes.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/TimeStamp.h" + +#include + +#include "jit/ExecutableAllocator.h" +#include "jit/JitCode.h" +#include "jit/ProcessExecutableMemory.h" +#include "vm/JSContext.h" +#include "vm/Realm.h" +#include "vm/Runtime.h" + +namespace js::jit { + +// This class ensures JIT code is executable on its destruction. Creators +// must call makeWritable(), and not attempt to write to the buffer if it fails. +// +// AutoWritableJitCodeFallible may only fail to make code writable; it cannot +// fail to make JIT code executable (because the creating code has no chance to +// recover from a failed destructor). +class MOZ_RAII AutoWritableJitCodeFallible { + JSRuntime* rt_; + void* addr_; + size_t size_; + + public: + AutoWritableJitCodeFallible(JSRuntime* rt, void* addr, size_t size) + : rt_(rt), addr_(addr), size_(size) { + rt_->toggleAutoWritableJitCodeActive(true); + } + + AutoWritableJitCodeFallible(void* addr, size_t size) + : AutoWritableJitCodeFallible(TlsContext.get()->runtime(), addr, size) {} + + explicit AutoWritableJitCodeFallible(JitCode* code) + : AutoWritableJitCodeFallible(code->runtimeFromMainThread(), code->raw(), + code->bufferSize()) {} + + [[nodiscard]] bool makeWritable() { + return ExecutableAllocator::makeWritable(addr_, size_); + } + + ~AutoWritableJitCodeFallible() { + mozilla::TimeStamp startTime = mozilla::TimeStamp::Now(); + auto timer = mozilla::MakeScopeExit([&] { + if (Realm* realm = rt_->mainContextFromOwnThread()->realm()) { + realm->timers.protectTime += mozilla::TimeStamp::Now() - startTime; + } + }); + + if (!ExecutableAllocator::makeExecutableAndFlushICache(addr_, size_)) { + MOZ_CRASH(); + } + rt_->toggleAutoWritableJitCodeActive(false); + } +}; + +// Infallible variant of AutoWritableJitCodeFallible, ensures writable during +// construction +class MOZ_RAII AutoWritableJitCode : private AutoWritableJitCodeFallible { + public: + AutoWritableJitCode(JSRuntime* rt, void* addr, size_t size) + : AutoWritableJitCodeFallible(rt, addr, size) { + MOZ_RELEASE_ASSERT(makeWritable()); + } + + AutoWritableJitCode(void* addr, size_t size) + : AutoWritableJitCode(TlsContext.get()->runtime(), addr, size) {} + + explicit AutoWritableJitCode(JitCode* code) + : AutoWritableJitCode(code->runtimeFromMainThread(), code->raw(), + code->bufferSize()) {} +}; + +} // namespace js::jit + +#endif /* jit_AutoWritableJitCode_h */ diff --git a/js/src/jit/BacktrackingAllocator.cpp b/js/src/jit/BacktrackingAllocator.cpp new file mode 100644 index 0000000000..d93431795a --- /dev/null +++ b/js/src/jit/BacktrackingAllocator.cpp @@ -0,0 +1,4676 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/////////////////////////////////////////////////////////////////////////////// +// // +// Documentation. Code starts about 670 lines down from here. // +// // +/////////////////////////////////////////////////////////////////////////////// + +// [SMDOC] An overview of Ion's register allocator +// +// The intent of this documentation is to give maintainers a map with which to +// navigate the allocator. As further understanding is obtained, it should be +// added to this overview. +// +// Where possible, invariants are stated and are marked "(INVAR)". Many +// details are omitted because their workings are currently unknown. In +// particular, this overview doesn't explain how Intel-style "modify" (tied) +// operands are handled. Facts or invariants that are speculative -- believed +// to be true, but not verified at the time of writing -- are marked "(SPEC)". +// +// The various concepts are interdependent, so a single forwards reading of the +// following won't make much sense. Many concepts are explained only after +// they are mentioned. +// +// Where possible examples are shown. Without those the description is +// excessively abstract. +// +// Names of the form ::name mean BacktrackingAllocator::name. +// +// The description falls into two sections: +// +// * Section 1: A tour of the data structures +// * Section 2: The core allocation loop, and bundle splitting +// +// The allocator sometimes produces poor allocations, with excessive spilling +// and register-to-register moves (bugs 1752520, bug 1714280 bug 1746596). +// Work in bug 1752582 shows we can get better quality allocations from this +// framework without having to make any large (conceptual) changes, by having +// better splitting heuristics. +// +// At https://bugzilla.mozilla.org/show_bug.cgi?id=1758274#c17 +// (https://bugzilla.mozilla.org/attachment.cgi?id=9288467) is a document +// written at the same time as these comments. It describes some improvements +// we could make to our splitting heuristics, particularly in the presence of +// loops and calls, and shows why the current implementation sometimes produces +// excessive spilling. It builds on the commentary in this SMDOC. +// +// +// Top level pipeline +// ~~~~~~~~~~~~~~~~~~ +// There are three major phases in allocation. They run sequentially, at a +// per-function granularity. +// +// (1) Liveness analysis and bundle formation +// (2) Bundle allocation and last-chance allocation +// (3) Rewriting the function to create MoveGroups and to "install" +// the allocation +// +// The input language (LIR) is in SSA form, and phases (1) and (3) depend on +// that SSAness. Without it the allocator wouldn't work. +// +// The top level function is ::go. The phases are divided into functions as +// follows: +// +// (1) ::buildLivenessInfo, ::mergeAndQueueRegisters +// (2) ::processBundle, ::tryAllocatingRegistersForSpillBundles, +// ::pickStackSlots +// (3) ::createMoveGroupsFromLiveRangeTransitions, ::installAllocationsInLIR, +// ::populateSafepoints, ::annotateMoveGroups +// +// The code in this file is structured as much as possible in the same sequence +// as flow through the pipeline. Hence, top level function ::go is right at +// the end. Where a function depends on helper function(s), the helpers appear +// first. +// +// +// ======================================================================== +// ==== ==== +// ==== Section 1: A tour of the data structures ==== +// ==== ==== +// ======================================================================== +// +// Here are the key data structures necessary for understanding what follows. +// +// Some basic data structures +// ~~~~~~~~~~~~~~~~~~~~~~~~~~ +// +// CodePosition +// ------------ +// A CodePosition is an unsigned 32-bit int that indicates an instruction index +// in the incoming LIR. Each LIR actually has two code positions, one to +// denote the "input" point (where, one might imagine, the operands are read, +// at least useAtStart ones) and the "output" point, where operands are +// written. Eg: +// +// Block 0 [successor 2] [successor 1] +// 2-3 WasmParameter [def v1:r14] +// 4-5 WasmCall [use v1:F:r14] +// 6-7 WasmLoadTls [def v2] [use v1:R] +// 8-9 WasmNullConstant [def v3] +// 10-11 Compare [def v4] [use v2:R] [use v3:A] +// 12-13 TestIAndBranch [use v4:R] +// +// So for example the WasmLoadTls insn has its input CodePosition as 6 and +// output point as 7. Input points are even numbered, output points are odd +// numbered. CodePositions 0 and 1 never appear, because LIR instruction IDs +// start at 1. Indeed, CodePosition 0 is assumed to be invalid and hence is +// used as a marker for "unusual conditions" in various places. +// +// Phi nodes exist in the instruction stream too. They always appear at the +// start of blocks (of course) (SPEC), but their start and end points are +// printed for the group as a whole. This is to emphasise that they are really +// parallel assignments and that printing them sequentially would misleadingly +// imply that they are executed sequentially. Example: +// +// Block 6 [successor 7] [successor 8] +// 56-59 Phi [def v19] [use v2:A] [use v5:A] [use v13:A] +// 56-59 Phi [def v20] [use v7:A] [use v14:A] [use v12:A] +// 60-61 WasmLoadSlot [def v21] [use v1:R] +// 62-63 Compare [def v22] [use v20:R] [use v21:A] +// 64-65 TestIAndBranch [use v22:R] +// +// See that both Phis are printed with limits 56-59, even though they are +// stored in the LIR array like regular LIRs and so have code points 56-57 and +// 58-59 in reality. +// +// The process of allocation adds MoveGroup LIRs to the function. Each +// incoming LIR has its own private list of MoveGroups (actually, 3 lists; two +// for moves that conceptually take place before the instruction, and one for +// moves after it). Hence the CodePositions for LIRs (the "62-63", etc, above) +// do not change as a result of allocation. +// +// Virtual registers (vregs) in LIR +// -------------------------------- +// The MIR from which the LIR is derived, is standard SSA. That SSAness is +// carried through into the LIR (SPEC). In the examples here, LIR SSA names +// (virtual registers, a.k.a. vregs) are printed as "v". v0 never +// appears and is presumed to be a special value, perhaps "invalid" (SPEC). +// +// The allocator core has a type VirtualRegister, but this is private to the +// allocator and not part of the LIR. It carries far more information than +// merely the name of the vreg. The allocator creates one VirtualRegister +// structure for each vreg in the LIR. +// +// LDefinition and LUse +// -------------------- +// These are part of the incoming LIR. Each LIR instruction defines zero or +// more values, and contains one LDefinition for each defined value (SPEC). +// Each instruction has zero or more input operands, each of which has its own +// LUse (SPEC). +// +// Both LDefinition and LUse hold both a virtual register name and, in general, +// a real (physical) register identity. The incoming LIR has the real register +// fields unset, except in places where the incoming LIR has fixed register +// constraints (SPEC). Phase 3 of allocation will visit all of the +// LDefinitions and LUses so as to write into the real register fields the +// decisions made by the allocator. For LUses, this is done by overwriting the +// complete LUse with a different LAllocation, for example LStackSlot. That's +// possible because LUse is a child class of LAllocation. +// +// This action of reading and then later updating LDefinition/LUses is the core +// of the allocator's interface to the outside world. +// +// To make visiting of LDefinitions/LUses possible, the allocator doesn't work +// with LDefinition and LUse directly. Rather it has pointers to them +// (VirtualRegister::def_, UsePosition::use_). Hence Phase 3 can modify the +// LIR in-place. +// +// (INVARs, all SPEC): +// +// - The collective VirtualRegister::def_ values should be unique, and there +// should be a 1:1 mapping between the VirtualRegister::def_ values and the +// LDefinitions in the LIR. (So that the LIR LDefinition has exactly one +// VirtualRegister::def_ to track it). But only for the valid LDefinitions. +// If isBogusTemp() is true, the definition is invalid and doesn't have a +// vreg. +// +// - The same for uses: there must be a 1:1 correspondence between the +// CodePosition::use_ values and the LIR LUses. +// +// - The allocation process must preserve these 1:1 mappings. That implies +// (weaker) that the number of VirtualRegisters and of UsePositions must +// remain constant through allocation. (Eg: losing them would mean that some +// LIR def or use would necessarily not get annotated with its final +// allocation decision. Duplicating them would lead to the possibility of +// conflicting allocation decisions.) +// +// Other comments regarding LIR +// ---------------------------- +// The incoming LIR is structured into basic blocks and a CFG, as one would +// expect. These (insns, block boundaries, block edges etc) are available +// through the BacktrackingAllocator object. They are important for Phases 1 +// and 3 but not for Phase 2. +// +// Phase 3 "rewrites" the input LIR so as to "install" the final allocation. +// It has to insert MoveGroup instructions, but that isn't done by pushing them +// into the instruction array. Rather, each LIR has 3 auxiliary sets of +// MoveGroups (SPEC): two that "happen" conceptually before the LIR, and one +// that happens after it. The rewriter inserts MoveGroups into one of these 3 +// sets, and later code generation phases presumably insert the sets (suitably +// translated) into the final machine code (SPEC). +// +// +// Key data structures: LiveRange, VirtualRegister and LiveBundle +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// +// These three have central roles in allocation. Of them, LiveRange is the +// most central. VirtualRegister is conceptually important throughout, but +// appears less frequently in the allocator code. LiveBundle is important only +// in Phase 2 (where it is central) and at the end of Phase 1, but plays no +// role in Phase 3. +// +// It's important to understand that LiveRange and VirtualRegister correspond +// to concepts visible in the incoming LIR, which is in SSA form. LiveBundle +// by comparison is related to neither the structure of LIR nor its SSA +// properties. Instead, LiveBundle is an essentially adminstrative structure +// used to accelerate allocation and to implement a crude form of +// move-coalescing. +// +// VirtualRegisters and LiveRanges are (almost) static throughout the process, +// because they reflect aspects of the incoming LIR, which does not change. +// LiveBundles by contrast come and go; they are created, but may be split up +// into new bundles, and old ones abandoned. +// +// Each LiveRange is a member of two different linked lists, chained through +// fields registerLink and bundleLink. +// +// A VirtualRegister (described in detail below) has a list of LiveRanges that +// it "owns". These are chained through LiveRange::registerLink. +// +// A LiveBundle (also described below) also has a list LiveRanges that it +// "owns", chained through LiveRange::bundleLink. +// +// Hence each LiveRange is "owned" by one VirtualRegister and one LiveBundle. +// LiveRanges may have their owning LiveBundle changed as a result of +// splitting. By contrast a LiveRange stays with its "owning" VirtualRegister +// for ever. +// +// A few LiveRanges have no VirtualRegister. This is used to implement +// register spilling for calls. Each physical register that's not preserved +// across a call has a small range that covers the call. It is +// ::buildLivenessInfo that adds these small ranges. +// +// Iterating over every VirtualRegister in the system is a common operation and +// is straightforward because (somewhat redundantly?) the LIRGraph knows the +// number of vregs, and more importantly because BacktrackingAllocator::vregs +// is a vector of all VirtualRegisters. By contrast iterating over every +// LiveBundle in the system is more complex because there is no top-level +// registry of them. It is still possible though. See ::dumpLiveRangesByVReg +// and ::dumpLiveRangesByBundle for example code. +// +// LiveRange +// --------- +// Fundamentally, a LiveRange (often written just "range") is a request for +// storage of a LIR vreg for some contiguous sequence of LIRs. A LiveRange +// generally covers only a fraction of a vreg's overall lifetime, so multiple +// LiveRanges are generally needed to cover the whole lifetime. +// +// A LiveRange contains (amongst other things): +// +// * the vreg for which it is for, as a VirtualRegister* +// +// * the range of CodePositions for which it is for, as a LiveRange::Range +// +// * auxiliary information: +// +// - a boolean that indicates whether this LiveRange defines a value for the +// vreg. If so, that definition is regarded as taking place at the first +// CodePoint of the range. +// +// - a linked list of uses of the vreg within this range. Each use is a pair +// of a CodePosition and an LUse*. (INVAR): the uses are maintained in +// increasing order of CodePosition. Multiple uses at the same +// CodePosition are permitted, since that is necessary to represent an LIR +// that uses the same vreg in more than one of its operand slots. +// +// Some important facts about LiveRanges are best illustrated with examples: +// +// v25 75-82 { 75_def:R 78_v25:A 82_v25:A } +// +// This LiveRange is for vreg v25. The value is defined at CodePosition 75, +// with the LIR requiring that it be in a register. It is used twice at +// positions 78 and 82, both with no constraints (A meaning "any"). The range +// runs from position 75 to 82 inclusive. Note however that LiveRange::Range +// uses non-inclusive range ends; hence its .to field would be 83, not 82. +// +// v26 84-85 { 85_v26:R } +// +// This LiveRange is for vreg v26. Here, there's only a single use of it at +// position 85. Presumably it is defined in some other LiveRange. +// +// v19 74-79 { } +// +// This LiveRange is for vreg v19. There is no def and no uses, so at first +// glance this seems redundant. But it isn't: it still expresses a request for +// storage for v19 across 74-79, because Phase 1 regards v19 as being live in +// this range (meaning: having a value that, if changed in this range, would +// cause the program to fail). +// +// Other points: +// +// * (INVAR) Each vreg/VirtualRegister has at least one LiveRange. +// +// * (INVAR) Exactly one LiveRange of a vreg gives a definition for the value. +// All other LiveRanges must consist only of uses (including zero uses, for a +// "flow-though" range as mentioned above). This requirement follows from +// the fact that LIR is in SSA form. +// +// * It follows from this, that the LiveRanges for a VirtualRegister must form +// a tree, where the parent-child relationship is "control flows directly +// from a parent LiveRange (anywhere in the LiveRange) to a child LiveRange +// (start)". The entire tree carries only one value. This is a use of +// SSAness in the allocator which is fundamental: without SSA input, this +// design would not work. +// +// The root node (LiveRange) in the tree must be one that defines the value, +// and all other nodes must only use or be flow-throughs for the value. It's +// OK for LiveRanges in the tree to overlap, providing that there is a unique +// root node -- otherwise it would be unclear which LiveRange provides the +// value. +// +// The function ::createMoveGroupsFromLiveRangeTransitions runs after all +// LiveBundles have been allocated. It visits each VirtualRegister tree in +// turn. For every parent->child edge in a tree, it creates a MoveGroup that +// copies the value from the parent into the child -- this is how the +// allocator decides where to put MoveGroups. There are various other +// details not described here. +// +// * It's important to understand that a LiveRange carries no meaning about +// control flow beyond that implied by the SSA (hence, dominance) +// relationship between a def and its uses. In particular, there's no +// implication that execution "flowing into" the start of the range implies +// that it will "flow out" of the end. Or that any particular use will or +// will not be executed. +// +// * (very SPEC) Indeed, even if a range has a def, there's no implication that +// a use later in the range will have been immediately preceded by execution +// of the def. It could be that the def is executed, flow jumps somewhere +// else, and later jumps back into the middle of the range, where there are +// then some uses. +// +// * Uses of a vreg by a phi node are not mentioned in the use list of a +// LiveRange. The reasons for this are unknown, but it is speculated that +// this is because we don't need to know about phi uses where we use the list +// of positions. See comments on VirtualRegister::usedByPhi_. +// +// * Similarly, a definition of a vreg by a phi node is not regarded as being a +// definition point (why not?), at least as the view of +// LiveRange::hasDefinition_. +// +// * LiveRanges that nevertheless include a phi-defined value have their first +// point set to the first of the block of phis, even if the var isn't defined +// by that specific phi. Eg: +// +// Block 6 [successor 7] [successor 8] +// 56-59 Phi [def v19] [use v2:A] [use v5:A] [use v13:A] +// 56-59 Phi [def v20] [use v7:A] [use v14:A] [use v12:A] +// 60-61 WasmLoadSlot [def v21] [use v1:R] +// 62-63 Compare [def v22] [use v20:R] [use v21:A] +// +// The relevant live range for v20 is +// +// v20 56-65 { 63_v20:R } +// +// Observe that it starts at 56, not 58. +// +// VirtualRegister +// --------------- +// Each VirtualRegister is associated with an SSA value created by the LIR. +// Fundamentally it is a container to hold all of the LiveRanges that together +// indicate where the value must be kept live. This is a linked list beginning +// at VirtualRegister::ranges_, and which, as described above, is chained +// through LiveRange::registerLink. The set of LiveRanges must logically form +// a tree, rooted at the LiveRange which defines the value. +// +// For adminstrative convenience, the linked list must contain the LiveRanges +// in order of increasing start point. +// +// There are various auxiliary fields, most importantly the LIR node and the +// associated LDefinition that define the value. +// +// It is OK, and quite common, for LiveRanges of a VirtualRegister to overlap. +// The effect will be that, in an overlapped area, there are two storage +// locations holding the value. This is OK -- although wasteful of storage +// resources -- because the SSAness means the value must be the same in both +// locations. Hence there's no questions like "which LiveRange holds the most +// up-to-date value?", since it's all just one value anyway. +// +// Note by contrast, it is *not* OK for the LiveRanges of a LiveBundle to +// overlap. +// +// LiveBundle +// ---------- +// Similar to VirtualRegister, a LiveBundle is also, fundamentally, a container +// for a set of LiveRanges. The set is stored as a linked list, rooted at +// LiveBundle::ranges_ and chained through LiveRange::bundleLink. +// +// However, the similarity ends there: +// +// * The LiveRanges in a LiveBundle absolutely must not overlap. They must +// indicate disjoint sets of CodePositions, and must be stored in the list in +// order of increasing CodePosition. Because of the no-overlap requirement, +// these ranges form a total ordering regardless of whether one uses the +// LiveRange::Range::from_ or ::to_ fields for comparison. +// +// * The LiveRanges in a LiveBundle can otherwise be entirely arbitrary and +// unrelated. They can be from different VirtualRegisters and can have no +// particular mutual significance w.r.t. the SSAness or structure of the +// input LIR. +// +// LiveBundles are the fundamental unit of allocation. The allocator attempts +// to find a single storage location that will work for all LiveRanges in the +// bundle. That's why the ranges must not overlap. If no such location can be +// found, the allocator may decide to split the bundle into multiple smaller +// bundles. Each of those may be allocated independently. +// +// The other really important field is LiveBundle::alloc_, indicating the +// chosen storage location. +// +// Here's an example, for a LiveBundle before it has been allocated: +// +// LB2(parent=none v3 8-21 { 16_v3:A } ## v3 24-25 { 25_v3:F:xmm0 }) +// +// LB merely indicates "LiveBundle", and the 2 is the debugId_ value (see +// below). This bundle has two LiveRanges +// +// v3 8-21 { 16_v3:A } +// v3 24-25 { 25_v3:F:xmm0 } +// +// both of which (coincidentally) are for the same VirtualRegister, v3.The +// second LiveRange has a fixed use in `xmm0`, whilst the first one doesn't +// care (A meaning "any location") so the allocator *could* choose `xmm0` for +// the bundle as a whole. +// +// One might ask: why bother with LiveBundle at all? After all, it would be +// possible to get correct allocations by allocating each LiveRange +// individually, then leaving ::createMoveGroupsFromLiveRangeTransitions to add +// MoveGroups to join up LiveRanges that form each SSA value tree (that is, +// LiveRanges belonging to each VirtualRegister). +// +// There are two reasons: +// +// (1) By putting multiple LiveRanges into each LiveBundle, we can end up with +// many fewer LiveBundles than LiveRanges. Since the cost of allocating a +// LiveBundle is substantially less than the cost of allocating each of its +// LiveRanges individually, the allocator will run faster. +// +// (2) It provides a crude form of move-coalescing. There are situations where +// it would be beneficial, although not mandatory, to have two LiveRanges +// assigned to the same storage unit. Most importantly: (a) LiveRanges +// that form all of the inputs, and the output, of a phi node. (b) +// LiveRanges for the output and first-operand values in the case where we +// are targetting Intel-style instructions. +// +// In such cases, if the bundle can be allocated as-is, then no extra moves +// are necessary. If not (and the bundle is split), then +// ::createMoveGroupsFromLiveRangeTransitions will later fix things up by +// inserting MoveGroups in the right places. +// +// Merging of LiveRanges into LiveBundles is done in Phase 1, by +// ::mergeAndQueueRegisters, after liveness analysis (which generates only +// LiveRanges). +// +// For the bundle mentioned above, viz +// +// LB2(parent=none v3 8-21 { 16_v3:A } ## v3 24-25 { 25_v3:F:xmm0 }) +// +// the allocator did not in the end choose to allocate it to `xmm0`. Instead +// it was split into two bundles, LB6 (a "spill parent", or root node in the +// trees described above), and LB9, a leaf node, that points to its parent LB6: +// +// LB6(parent=none v3 8-21 %xmm1.s { 16_v3:A } ## v3 24-25 %xmm1.s { }) +// LB9(parent=LB6 v3 24-25 %xmm0.s { 25_v3:F:rax }) +// +// Note that both bundles now have an allocation, and that is printed, +// redundantly, for each LiveRange in the bundle -- hence the repeated +// `%xmm1.s` in the lines above. Since all LiveRanges in a LiveBundle must be +// allocated to the same storage location, we never expect to see output like +// this +// +// LB6(parent=none v3 8-21 %xmm1.s { 16_v3:A } ## v3 24-25 %xmm2.s { }) +// +// and that is in any case impossible, since a LiveRange doesn't have an +// LAllocation field. Instead it has a pointer to its LiveBundle, and the +// LAllocation lives in the LiveBundle. +// +// For the resulting allocation (LB6 and LB9), all the LiveRanges are use-only +// or flow-through. There are no definitions. But they are all for the same +// VirtualReg, v3, so they all have the same value. An important question is +// where they "get their value from". The answer is that +// ::createMoveGroupsFromLiveRangeTransitions will have to insert suitable +// MoveGroups so that each LiveRange for v3 can "acquire" the value from a +// previously-executed LiveRange, except for the range that defines it. The +// defining LiveRange is not visible here; either it is in some other +// LiveBundle, not shown, or (more likely) the value is defined by a phi-node, +// in which case, as mentioned previously, it is not shown as having an +// explicit definition in any LiveRange. +// +// LiveBundles also have a `SpillSet* spill_` field (see below) and a +// `LiveBundle* spillParent_`. The latter is used to ensure that all bundles +// originating from an "original" bundle share the same spill slot. The +// `spillParent_` pointers can be thought of creating a 1-level tree, where +// each node points at its parent. Since the tree can be only 1-level, the +// following invariant (INVAR) must be maintained: +// +// * if a LiveBundle has a non-null spillParent_ field, then it is a leaf node, +// and no other LiveBundle can point at this one. +// +// * else (it has a null spillParent_ field) it is a root node, and so other +// LiveBundles may point at it. +// +// When compiled with JS_JITSPEW, LiveBundle has a 32-bit `debugId_` field. +// This is used only for debug printing, and makes it easier to see +// parent-child relationships induced by the `spillParent_` pointers. +// +// The "life cycle" of LiveBundles is described in Section 2 below. +// +// +// Secondary data structures: SpillSet, Requirement +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// +// SpillSet +// -------- +// A SpillSet is a container for a set of LiveBundles that have been spilled, +// all of which are assigned the same spill slot. The set is represented as a +// vector of points to LiveBundles. SpillSet also contains the identity of the +// spill slot (its LAllocation). +// +// A LiveBundle, if it is to be spilled, has a pointer to the relevant +// SpillSet, and the SpillSet in turn has a pointer back to the LiveBundle in +// its vector thereof. So LiveBundles (that are to be spilled) and SpillSets +// point at each other. +// +// (SPEC) LiveBundles that are not to be spilled (or for which the decision has +// yet to be made, have their SpillSet pointers as null. (/SPEC) +// +// Requirement +// ----------- +// Requirements are used transiently during the main allocation loop. It +// summarises the set of constraints on storage location (must be any register, +// must be this specific register, must be stack, etc) for a LiveBundle. This +// is so that the main allocation loop knows what kind of storage location it +// must choose in order to satisfy all of the defs and uses within the bundle. +// +// What Requirement provides is (a) a partially ordered set of locations, and +// (b) a constraint-merging method `merge`. +// +// Requirement needs a rewrite (and, in fact, that has already happened in +// un-landed code in bug 1758274) for the following reasons: +// +// * it's potentially buggy (bug 1761654), although that doesn't currently +// affect us, for reasons which are unclear. +// +// * the partially ordered set has a top point, meaning "no constraint", but it +// doesn't have a corresponding bottom point, meaning "impossible +// constraints". (So it's a partially ordered set, but not a lattice). This +// leads to awkward coding in some places, which would be simplified if there +// were an explicit way to indicate "impossible constraint". +// +// +// Some ASCII art +// ~~~~~~~~~~~~~~ +// +// Here's some not-very-good ASCII art that tries to summarise the data +// structures that persist for the entire allocation of a function: +// +// BacktrackingAllocator +// | +// (vregs) +// | +// v +// | +// VirtualRegister -->--(ins)--> LNode +// | | `->--(def)--> LDefinition +// v ^ +// | | +// (ranges) | +// | (vreg) +// `--v->--. | ,-->--v-->-------------->--v-->--. ,--NULL +// \ | / \ / +// LiveRange LiveRange LiveRange +// / | \ / \. +// ,--b->--' / `-->--b-->--' `--NULL +// | (bundle) +// ^ / +// | v +// (ranges) / +// | / +// LiveBundle --s-->- LiveBundle +// | \ / | +// | \ / | +// (spill) ^ ^ (spill) +// | \ / | +// v \ / ^ +// | (list) | +// \ | / +// `--->---> SpillSet <--' +// +// --b-- LiveRange::bundleLink: links in the list of LiveRanges that belong to +// a LiveBundle +// +// --v-- LiveRange::registerLink: links in the list of LiveRanges that belong +// to a VirtualRegister +// +// --s-- LiveBundle::spillParent: a possible link to my "spill parent bundle" +// +// +// * LiveRange is in the center. Each LiveRange is a member of two different +// linked lists, the --b-- list and the --v-- list. +// +// * VirtualRegister has a pointer `ranges` that points to the start of its +// --v-- list of LiveRanges. +// +// * LiveBundle has a pointer `ranges` that points to the start of its --b-- +// list of LiveRanges. +// +// * LiveRange points back at both its owning VirtualRegister (`vreg`) and its +// owning LiveBundle (`bundle`). +// +// * LiveBundle has a pointer --s-- `spillParent`, which may be null, to its +// conceptual "spill parent bundle", as discussed in detail above. +// +// * LiveBundle has a pointer `spill` to its SpillSet. +// +// * SpillSet has a vector `list` of pointers back to the LiveBundles that +// point at it. +// +// * VirtualRegister has pointers `ins` to the LNode that defines the value and +// `def` to the LDefinition within that LNode. +// +// * BacktrackingAllocator has a vector `vregs` of pointers to all the +// VirtualRegisters for the function. There is no equivalent top-level table +// of all the LiveBundles for the function. +// +// Note that none of these pointers are "owning" in the C++-storage-management +// sense. Rather, everything is stored in single arena which is freed when +// compilation of the function is complete. For this reason, +// BacktrackingAllocator.{h,cpp} is almost completely free of the usual C++ +// storage-management artefacts one would normally expect to see. +// +// +// ======================================================================== +// ==== ==== +// ==== Section 2: The core allocation loop, and bundle splitting ==== +// ==== ==== +// ======================================================================== +// +// Phase 1 of the allocator (described at the start of this SMDOC) computes +// live ranges, merges them into bundles, and places the bundles in a priority +// queue ::allocationQueue, ordered by what ::computePriority computes. +// +// +// The allocation loops +// ~~~~~~~~~~~~~~~~~~~~ +// The core of the allocation machinery consists of two loops followed by a +// call to ::pickStackSlots. The latter is uninteresting. The two loops live +// in ::go and are documented in detail there. +// +// +// Bundle splitting +// ~~~~~~~~~~~~~~~~ +// If the first of the two abovementioned loops cannot find a register for a +// bundle, either directly or as a result of evicting conflicting bundles, then +// it will have to either split or spill the bundle. The entry point to the +// split/spill subsystem is ::chooseBundleSplit. See comments there. + +/////////////////////////////////////////////////////////////////////////////// +// // +// End of documentation // +// // +/////////////////////////////////////////////////////////////////////////////// + +#include "jit/BacktrackingAllocator.h" + +#include + +#include "jit/BitSet.h" +#include "jit/CompileInfo.h" +#include "js/Printf.h" + +using namespace js; +using namespace js::jit; + +using mozilla::DebugOnly; + +// This is a big, complex file. Code is grouped into various sections, each +// preceded by a box comment. Sections not marked as "Misc helpers" are +// pretty much the top level flow, and are presented roughly in the same order +// in which the allocation pipeline operates. BacktrackingAllocator::go, +// right at the end of the file, is a good starting point. + +/////////////////////////////////////////////////////////////////////////////// +// // +// Misc helpers: linked-list management // +// // +/////////////////////////////////////////////////////////////////////////////// + +static inline bool SortBefore(UsePosition* a, UsePosition* b) { + return a->pos <= b->pos; +} + +static inline bool SortBefore(LiveRange::BundleLink* a, + LiveRange::BundleLink* b) { + LiveRange* rangea = LiveRange::get(a); + LiveRange* rangeb = LiveRange::get(b); + MOZ_ASSERT(!rangea->intersects(rangeb)); + return rangea->from() < rangeb->from(); +} + +static inline bool SortBefore(LiveRange::RegisterLink* a, + LiveRange::RegisterLink* b) { + return LiveRange::get(a)->from() <= LiveRange::get(b)->from(); +} + +template +static inline void InsertSortedList(InlineForwardList& list, T* value) { + if (list.empty()) { + list.pushFront(value); + return; + } + + if (SortBefore(list.back(), value)) { + list.pushBack(value); + return; + } + + T* prev = nullptr; + for (InlineForwardListIterator iter = list.begin(); iter; iter++) { + if (SortBefore(value, *iter)) { + break; + } + prev = *iter; + } + + if (prev) { + list.insertAfter(prev, value); + } else { + list.pushFront(value); + } +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Misc helpers: methods for class SpillSet // +// // +/////////////////////////////////////////////////////////////////////////////// + +void SpillSet::setAllocation(LAllocation alloc) { + for (size_t i = 0; i < numSpilledBundles(); i++) { + spilledBundle(i)->setAllocation(alloc); + } +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Misc helpers: methods for class LiveRange // +// // +/////////////////////////////////////////////////////////////////////////////// + +static size_t SpillWeightFromUsePolicy(LUse::Policy policy) { + switch (policy) { + case LUse::ANY: + return 1000; + + case LUse::REGISTER: + case LUse::FIXED: + return 2000; + + default: + return 0; + } +} + +inline void LiveRange::noteAddedUse(UsePosition* use) { + LUse::Policy policy = use->usePolicy(); + usesSpillWeight_ += SpillWeightFromUsePolicy(policy); + if (policy == LUse::FIXED) { + ++numFixedUses_; + } +} + +inline void LiveRange::noteRemovedUse(UsePosition* use) { + LUse::Policy policy = use->usePolicy(); + usesSpillWeight_ -= SpillWeightFromUsePolicy(policy); + if (policy == LUse::FIXED) { + --numFixedUses_; + } + MOZ_ASSERT_IF(!hasUses(), !usesSpillWeight_ && !numFixedUses_); +} + +void LiveRange::addUse(UsePosition* use) { + MOZ_ASSERT(covers(use->pos)); + InsertSortedList(uses_, use); + noteAddedUse(use); +} + +UsePosition* LiveRange::popUse() { + UsePosition* ret = uses_.popFront(); + noteRemovedUse(ret); + return ret; +} + +void LiveRange::tryToMoveDefAndUsesInto(LiveRange* other) { + MOZ_ASSERT(&other->vreg() == &vreg()); + MOZ_ASSERT(this != other); + + // Move over all uses which fit in |other|'s boundaries. + for (UsePositionIterator iter = usesBegin(); iter;) { + UsePosition* use = *iter; + if (other->covers(use->pos)) { + uses_.removeAndIncrement(iter); + noteRemovedUse(use); + other->addUse(use); + } else { + iter++; + } + } + + // Distribute the definition to |other| as well, if possible. + if (hasDefinition() && from() == other->from()) { + other->setHasDefinition(); + } +} + +bool LiveRange::contains(LiveRange* other) const { + return from() <= other->from() && to() >= other->to(); +} + +void LiveRange::intersect(LiveRange* other, Range* pre, Range* inside, + Range* post) const { + MOZ_ASSERT(pre->empty() && inside->empty() && post->empty()); + + CodePosition innerFrom = from(); + if (from() < other->from()) { + if (to() < other->from()) { + *pre = range_; + return; + } + *pre = Range(from(), other->from()); + innerFrom = other->from(); + } + + CodePosition innerTo = to(); + if (to() > other->to()) { + if (from() >= other->to()) { + *post = range_; + return; + } + *post = Range(other->to(), to()); + innerTo = other->to(); + } + + if (innerFrom != innerTo) { + *inside = Range(innerFrom, innerTo); + } +} + +bool LiveRange::intersects(LiveRange* other) const { + Range pre, inside, post; + intersect(other, &pre, &inside, &post); + return !inside.empty(); +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Misc helpers: methods for class LiveBundle // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifdef DEBUG +size_t LiveBundle::numRanges() const { + size_t count = 0; + for (LiveRange::BundleLinkIterator iter = rangesBegin(); iter; iter++) { + count++; + } + return count; +} +#endif + +LiveRange* LiveBundle::rangeFor(CodePosition pos) const { + for (LiveRange::BundleLinkIterator iter = rangesBegin(); iter; iter++) { + LiveRange* range = LiveRange::get(*iter); + if (range->covers(pos)) { + return range; + } + } + return nullptr; +} + +void LiveBundle::addRange(LiveRange* range) { + MOZ_ASSERT(!range->bundle()); + range->setBundle(this); + InsertSortedList(ranges_, &range->bundleLink); +} + +bool LiveBundle::addRange(TempAllocator& alloc, VirtualRegister* vreg, + CodePosition from, CodePosition to) { + LiveRange* range = LiveRange::FallibleNew(alloc, vreg, from, to); + if (!range) { + return false; + } + addRange(range); + return true; +} + +bool LiveBundle::addRangeAndDistributeUses(TempAllocator& alloc, + LiveRange* oldRange, + CodePosition from, CodePosition to) { + LiveRange* range = LiveRange::FallibleNew(alloc, &oldRange->vreg(), from, to); + if (!range) { + return false; + } + addRange(range); + oldRange->tryToMoveDefAndUsesInto(range); + return true; +} + +LiveRange* LiveBundle::popFirstRange() { + LiveRange::BundleLinkIterator iter = rangesBegin(); + if (!iter) { + return nullptr; + } + + LiveRange* range = LiveRange::get(*iter); + ranges_.removeAt(iter); + + range->setBundle(nullptr); + return range; +} + +void LiveBundle::removeRange(LiveRange* range) { + for (LiveRange::BundleLinkIterator iter = rangesBegin(); iter; iter++) { + LiveRange* existing = LiveRange::get(*iter); + if (existing == range) { + ranges_.removeAt(iter); + return; + } + } + MOZ_CRASH(); +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Misc helpers: methods for class VirtualRegister // +// // +/////////////////////////////////////////////////////////////////////////////// + +bool VirtualRegister::addInitialRange(TempAllocator& alloc, CodePosition from, + CodePosition to, size_t* numRanges) { + MOZ_ASSERT(from < to); + + // Mark [from,to) as a live range for this register during the initial + // liveness analysis, coalescing with any existing overlapping ranges. + + // On some pathological graphs there might be a huge number of different + // live ranges. Allow non-overlapping live range to be merged if the + // number of ranges exceeds the cap below. + static const size_t CoalesceLimit = 100000; + + LiveRange* prev = nullptr; + LiveRange* merged = nullptr; + for (LiveRange::RegisterLinkIterator iter(rangesBegin()); iter;) { + LiveRange* existing = LiveRange::get(*iter); + + if (from > existing->to() && *numRanges < CoalesceLimit) { + // The new range should go after this one. + prev = existing; + iter++; + continue; + } + + if (to.next() < existing->from()) { + // The new range should go before this one. + break; + } + + if (!merged) { + // This is the first old range we've found that overlaps the new + // range. Extend this one to cover its union with the new range. + merged = existing; + + if (from < existing->from()) { + existing->setFrom(from); + } + if (to > existing->to()) { + existing->setTo(to); + } + + // Continue searching to see if any other old ranges can be + // coalesced with the new merged range. + iter++; + continue; + } + + // Coalesce this range into the previous range we merged into. + MOZ_ASSERT(existing->from() >= merged->from()); + if (existing->to() > merged->to()) { + merged->setTo(existing->to()); + } + + MOZ_ASSERT(!existing->hasDefinition()); + existing->tryToMoveDefAndUsesInto(merged); + MOZ_ASSERT(!existing->hasUses()); + + ranges_.removeAndIncrement(iter); + } + + if (!merged) { + // The new range does not overlap any existing range for the vreg. + LiveRange* range = LiveRange::FallibleNew(alloc, this, from, to); + if (!range) { + return false; + } + + if (prev) { + ranges_.insertAfter(&prev->registerLink, &range->registerLink); + } else { + ranges_.pushFront(&range->registerLink); + } + + (*numRanges)++; + } + + return true; +} + +void VirtualRegister::addInitialUse(UsePosition* use) { + LiveRange::get(*rangesBegin())->addUse(use); +} + +void VirtualRegister::setInitialDefinition(CodePosition from) { + LiveRange* first = LiveRange::get(*rangesBegin()); + MOZ_ASSERT(from >= first->from()); + first->setFrom(from); + first->setHasDefinition(); +} + +LiveRange* VirtualRegister::rangeFor(CodePosition pos, + bool preferRegister /* = false */) const { + LiveRange* found = nullptr; + for (LiveRange::RegisterLinkIterator iter = rangesBegin(); iter; iter++) { + LiveRange* range = LiveRange::get(*iter); + if (range->covers(pos)) { + if (!preferRegister || range->bundle()->allocation().isRegister()) { + return range; + } + if (!found) { + found = range; + } + } + } + return found; +} + +void VirtualRegister::addRange(LiveRange* range) { + InsertSortedList(ranges_, &range->registerLink); +} + +void VirtualRegister::removeRange(LiveRange* range) { + for (LiveRange::RegisterLinkIterator iter = rangesBegin(); iter; iter++) { + LiveRange* existing = LiveRange::get(*iter); + if (existing == range) { + ranges_.removeAt(iter); + return; + } + } + MOZ_CRASH(); +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Misc helpers: queries about uses // +// // +/////////////////////////////////////////////////////////////////////////////// + +static inline LDefinition* FindReusingDefOrTemp(LNode* node, + LAllocation* alloc) { + if (node->isPhi()) { + MOZ_ASSERT(node->toPhi()->numDefs() == 1); + MOZ_ASSERT(node->toPhi()->getDef(0)->policy() != + LDefinition::MUST_REUSE_INPUT); + return nullptr; + } + + LInstruction* ins = node->toInstruction(); + + for (size_t i = 0; i < ins->numDefs(); i++) { + LDefinition* def = ins->getDef(i); + if (def->policy() == LDefinition::MUST_REUSE_INPUT && + ins->getOperand(def->getReusedInput()) == alloc) { + return def; + } + } + for (size_t i = 0; i < ins->numTemps(); i++) { + LDefinition* def = ins->getTemp(i); + if (def->policy() == LDefinition::MUST_REUSE_INPUT && + ins->getOperand(def->getReusedInput()) == alloc) { + return def; + } + } + return nullptr; +} + +bool BacktrackingAllocator::isReusedInput(LUse* use, LNode* ins, + bool considerCopy) { + if (LDefinition* def = FindReusingDefOrTemp(ins, use)) { + return considerCopy || !vregs[def->virtualRegister()].mustCopyInput(); + } + return false; +} + +bool BacktrackingAllocator::isRegisterUse(UsePosition* use, LNode* ins, + bool considerCopy) { + switch (use->usePolicy()) { + case LUse::ANY: + return isReusedInput(use->use(), ins, considerCopy); + + case LUse::REGISTER: + case LUse::FIXED: + return true; + + default: + return false; + } +} + +bool BacktrackingAllocator::isRegisterDefinition(LiveRange* range) { + if (!range->hasDefinition()) { + return false; + } + + VirtualRegister& reg = range->vreg(); + if (reg.ins()->isPhi()) { + return false; + } + + if (reg.def()->policy() == LDefinition::FIXED && + !reg.def()->output()->isRegister()) { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Misc helpers: atomic LIR groups // +// // +/////////////////////////////////////////////////////////////////////////////// + +// The following groupings contain implicit (invisible to ::buildLivenessInfo) +// value flows, and therefore no split points may be requested inside them. +// This is an otherwise unstated part of the contract between LIR generation +// and the allocator. +// +// (1) (any insn) ; OsiPoint +// +// [Further group definitions and supporting code to come, pending rework +// of the wasm atomic-group situation.] + +CodePosition RegisterAllocator::minimalDefEnd(LNode* ins) const { + // Compute the shortest interval that captures vregs defined by ins. + // Watch for instructions that are followed by an OSI point. + // If moves are introduced between the instruction and the OSI point then + // safepoint information for the instruction may be incorrect. + while (true) { + LNode* next = insData[ins->id() + 1]; + if (!next->isOsiPoint()) { + break; + } + ins = next; + } + + return outputOf(ins); +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Misc helpers: computation of bundle priorities and spill weights // +// // +/////////////////////////////////////////////////////////////////////////////// + +size_t BacktrackingAllocator::computePriority(LiveBundle* bundle) { + // The priority of a bundle is its total length, so that longer lived + // bundles will be processed before shorter ones (even if the longer ones + // have a low spill weight). See processBundle(). + size_t lifetimeTotal = 0; + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + lifetimeTotal += range->to() - range->from(); + } + + return lifetimeTotal; +} + +bool BacktrackingAllocator::minimalDef(LiveRange* range, LNode* ins) { + // Whether this is a minimal range capturing a definition at ins. + return (range->to() <= minimalDefEnd(ins).next()) && + ((!ins->isPhi() && range->from() == inputOf(ins)) || + range->from() == outputOf(ins)); +} + +bool BacktrackingAllocator::minimalUse(LiveRange* range, UsePosition* use) { + // Whether this is a minimal range capturing |use|. + LNode* ins = insData[use->pos]; + return (range->from() == inputOf(ins)) && + (range->to() == + (use->use()->usedAtStart() ? outputOf(ins) : outputOf(ins).next())); +} + +bool BacktrackingAllocator::minimalBundle(LiveBundle* bundle, bool* pfixed) { + LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); + LiveRange* range = LiveRange::get(*iter); + + if (!range->hasVreg()) { + *pfixed = true; + return true; + } + + // If a bundle contains multiple ranges, splitAtAllRegisterUses will split + // each range into a separate bundle. + if (++iter) { + return false; + } + + if (range->hasDefinition()) { + VirtualRegister& reg = range->vreg(); + if (pfixed) { + *pfixed = reg.def()->policy() == LDefinition::FIXED && + reg.def()->output()->isRegister(); + } + return minimalDef(range, reg.ins()); + } + + bool fixed = false, minimal = false, multiple = false; + + for (UsePositionIterator iter = range->usesBegin(); iter; iter++) { + if (iter != range->usesBegin()) { + multiple = true; + } + + switch (iter->usePolicy()) { + case LUse::FIXED: + if (fixed) { + return false; + } + fixed = true; + if (minimalUse(range, *iter)) { + minimal = true; + } + break; + + case LUse::REGISTER: + if (minimalUse(range, *iter)) { + minimal = true; + } + break; + + default: + break; + } + } + + // If a range contains a fixed use and at least one other use, + // splitAtAllRegisterUses will split each use into a different bundle. + if (multiple && fixed) { + minimal = false; + } + + if (pfixed) { + *pfixed = fixed; + } + return minimal; +} + +size_t BacktrackingAllocator::computeSpillWeight(LiveBundle* bundle) { + // Minimal bundles have an extremely high spill weight, to ensure they + // can evict any other bundles and be allocated to a register. + bool fixed; + if (minimalBundle(bundle, &fixed)) { + return fixed ? 2000000 : 1000000; + } + + size_t usesTotal = 0; + fixed = false; + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + + if (range->hasDefinition()) { + VirtualRegister& reg = range->vreg(); + if (reg.def()->policy() == LDefinition::FIXED && + reg.def()->output()->isRegister()) { + usesTotal += 2000; + fixed = true; + } else if (!reg.ins()->isPhi()) { + usesTotal += 2000; + } + } + + usesTotal += range->usesSpillWeight(); + if (range->numFixedUses() > 0) { + fixed = true; + } + } + + // Bundles with fixed uses are given a higher spill weight, since they must + // be allocated to a specific register. + if (testbed && fixed) { + usesTotal *= 2; + } + + // Compute spill weight as a use density, lowering the weight for long + // lived bundles with relatively few uses. + size_t lifetimeTotal = computePriority(bundle); + return lifetimeTotal ? usesTotal / lifetimeTotal : 0; +} + +size_t BacktrackingAllocator::maximumSpillWeight( + const LiveBundleVector& bundles) { + size_t maxWeight = 0; + for (size_t i = 0; i < bundles.length(); i++) { + maxWeight = std::max(maxWeight, computeSpillWeight(bundles[i])); + } + return maxWeight; +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Initialization of the allocator // +// // +/////////////////////////////////////////////////////////////////////////////// + +// This function pre-allocates and initializes as much global state as possible +// to avoid littering the algorithms with memory management cruft. +bool BacktrackingAllocator::init() { + if (!RegisterAllocator::init()) { + return false; + } + + liveIn = mir->allocate(graph.numBlockIds()); + if (!liveIn) { + return false; + } + + size_t numVregs = graph.numVirtualRegisters(); + if (!vregs.init(mir->alloc(), numVregs)) { + return false; + } + for (uint32_t i = 0; i < numVregs; i++) { + new (&vregs[i]) VirtualRegister(); + } + + // Build virtual register objects. + for (size_t i = 0; i < graph.numBlocks(); i++) { + if (mir->shouldCancel("Create data structures (main loop)")) { + return false; + } + + LBlock* block = graph.getBlock(i); + for (LInstructionIterator ins = block->begin(); ins != block->end(); + ins++) { + if (mir->shouldCancel("Create data structures (inner loop 1)")) { + return false; + } + + for (size_t j = 0; j < ins->numDefs(); j++) { + LDefinition* def = ins->getDef(j); + if (def->isBogusTemp()) { + continue; + } + vreg(def).init(*ins, def, /* isTemp = */ false); + } + + for (size_t j = 0; j < ins->numTemps(); j++) { + LDefinition* def = ins->getTemp(j); + if (def->isBogusTemp()) { + continue; + } + vreg(def).init(*ins, def, /* isTemp = */ true); + } + } + for (size_t j = 0; j < block->numPhis(); j++) { + LPhi* phi = block->getPhi(j); + LDefinition* def = phi->getDef(0); + vreg(def).init(phi, def, /* isTemp = */ false); + } + } + + LiveRegisterSet remainingRegisters(allRegisters_.asLiveSet()); + while (!remainingRegisters.emptyGeneral()) { + AnyRegister reg = AnyRegister(remainingRegisters.takeAnyGeneral()); + registers[reg.code()].allocatable = true; + } + while (!remainingRegisters.emptyFloat()) { + AnyRegister reg = + AnyRegister(remainingRegisters.takeAnyFloat()); + registers[reg.code()].allocatable = true; + } + + LifoAlloc* lifoAlloc = mir->alloc().lifoAlloc(); + for (size_t i = 0; i < AnyRegister::Total; i++) { + registers[i].reg = AnyRegister::FromCode(i); + registers[i].allocations.setAllocator(lifoAlloc); + } + + hotcode.setAllocator(lifoAlloc); + callRanges.setAllocator(lifoAlloc); + + // Partition the graph into hot and cold sections, for helping to make + // splitting decisions. Since we don't have any profiling data this is a + // crapshoot, so just mark the bodies of inner loops as hot and everything + // else as cold. + + LBlock* backedge = nullptr; + for (size_t i = 0; i < graph.numBlocks(); i++) { + LBlock* block = graph.getBlock(i); + + // If we see a loop header, mark the backedge so we know when we have + // hit the end of the loop. Don't process the loop immediately, so that + // if there is an inner loop we will ignore the outer backedge. + if (block->mir()->isLoopHeader()) { + backedge = block->mir()->backedge()->lir(); + } + + if (block == backedge) { + LBlock* header = block->mir()->loopHeaderOfBackedge()->lir(); + LiveRange* range = LiveRange::FallibleNew( + alloc(), nullptr, entryOf(header), exitOf(block).next()); + if (!range || !hotcode.insert(range)) { + return false; + } + } + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Liveness analysis // +// // +/////////////////////////////////////////////////////////////////////////////// + +// Helper for ::buildLivenessInfo +bool BacktrackingAllocator::addInitialFixedRange(AnyRegister reg, + CodePosition from, + CodePosition to) { + LiveRange* range = LiveRange::FallibleNew(alloc(), nullptr, from, to); + if (!range) { + return false; + } + LiveRangePlus rangePlus(range); + return registers[reg.code()].allocations.insert(rangePlus); +} + +// Helper for ::buildLivenessInfo +#ifdef DEBUG +// Returns true iff ins has a def/temp reusing the input allocation. +static bool IsInputReused(LInstruction* ins, LUse* use) { + for (size_t i = 0; i < ins->numDefs(); i++) { + if (ins->getDef(i)->policy() == LDefinition::MUST_REUSE_INPUT && + ins->getOperand(ins->getDef(i)->getReusedInput())->toUse() == use) { + return true; + } + } + + for (size_t i = 0; i < ins->numTemps(); i++) { + if (ins->getTemp(i)->policy() == LDefinition::MUST_REUSE_INPUT && + ins->getOperand(ins->getTemp(i)->getReusedInput())->toUse() == use) { + return true; + } + } + + return false; +} +#endif + +/* + * This function builds up liveness ranges for all virtual registers + * defined in the function. + * + * The algorithm is based on the one published in: + * + * Wimmer, Christian, and Michael Franz. "Linear Scan Register Allocation on + * SSA Form." Proceedings of the International Symposium on Code Generation + * and Optimization. Toronto, Ontario, Canada, ACM. 2010. 170-79. PDF. + * + * The algorithm operates on blocks ordered such that dominators of a block + * are before the block itself, and such that all blocks of a loop are + * contiguous. It proceeds backwards over the instructions in this order, + * marking registers live at their uses, ending their live ranges at + * definitions, and recording which registers are live at the top of every + * block. To deal with loop backedges, registers live at the beginning of + * a loop gain a range covering the entire loop. + */ +bool BacktrackingAllocator::buildLivenessInfo() { + JitSpew(JitSpew_RegAlloc, "Beginning liveness analysis"); + + Vector loopWorkList; + BitSet loopDone(graph.numBlockIds()); + if (!loopDone.init(alloc())) { + return false; + } + + size_t numRanges = 0; + + for (size_t i = graph.numBlocks(); i > 0; i--) { + if (mir->shouldCancel("Build Liveness Info (main loop)")) { + return false; + } + + LBlock* block = graph.getBlock(i - 1); + MBasicBlock* mblock = block->mir(); + + BitSet& live = liveIn[mblock->id()]; + new (&live) BitSet(graph.numVirtualRegisters()); + if (!live.init(alloc())) { + return false; + } + + // Propagate liveIn from our successors to us. + for (size_t i = 0; i < mblock->lastIns()->numSuccessors(); i++) { + MBasicBlock* successor = mblock->lastIns()->getSuccessor(i); + // Skip backedges, as we fix them up at the loop header. + if (mblock->id() < successor->id()) { + live.insertAll(liveIn[successor->id()]); + } + } + + // Add successor phis. + if (mblock->successorWithPhis()) { + LBlock* phiSuccessor = mblock->successorWithPhis()->lir(); + for (unsigned int j = 0; j < phiSuccessor->numPhis(); j++) { + LPhi* phi = phiSuccessor->getPhi(j); + LAllocation* use = phi->getOperand(mblock->positionInPhiSuccessor()); + uint32_t reg = use->toUse()->virtualRegister(); + live.insert(reg); + vreg(use).setUsedByPhi(); + } + } + + // Registers are assumed alive for the entire block, a define shortens + // the range to the point of definition. + for (BitSet::Iterator liveRegId(live); liveRegId; ++liveRegId) { + if (!vregs[*liveRegId].addInitialRange(alloc(), entryOf(block), + exitOf(block).next(), &numRanges)) + return false; + } + + // Shorten the front end of ranges for live variables to their point of + // definition, if found. + for (LInstructionReverseIterator ins = block->rbegin(); + ins != block->rend(); ins++) { + // Calls may clobber registers, so force a spill and reload around the + // callsite. + if (ins->isCall()) { + for (AnyRegisterIterator iter(allRegisters_.asLiveSet()); iter.more(); + ++iter) { + bool found = false; + for (size_t i = 0; i < ins->numDefs(); i++) { + if (ins->getDef(i)->isFixed() && + ins->getDef(i)->output()->aliases(LAllocation(*iter))) { + found = true; + break; + } + } + // If this register doesn't have an explicit def above, mark + // it as clobbered by the call unless it is actually + // call-preserved. + if (!found && !ins->isCallPreserved(*iter)) { + if (!addInitialFixedRange(*iter, outputOf(*ins), + outputOf(*ins).next())) { + return false; + } + } + } + + CallRange* callRange = new (alloc().fallible()) + CallRange(outputOf(*ins), outputOf(*ins).next()); + if (!callRange) { + return false; + } + + callRangesList.pushFront(callRange); + if (!callRanges.insert(callRange)) { + return false; + } + } + + for (size_t i = 0; i < ins->numDefs(); i++) { + LDefinition* def = ins->getDef(i); + if (def->isBogusTemp()) { + continue; + } + + CodePosition from = outputOf(*ins); + + if (def->policy() == LDefinition::MUST_REUSE_INPUT) { + // MUST_REUSE_INPUT is implemented by allocating an output + // register and moving the input to it. Register hints are + // used to avoid unnecessary moves. We give the input an + // LUse::ANY policy to avoid allocating a register for the + // input. + LUse* inputUse = ins->getOperand(def->getReusedInput())->toUse(); + MOZ_ASSERT(inputUse->policy() == LUse::REGISTER); + MOZ_ASSERT(inputUse->usedAtStart()); + *inputUse = LUse(inputUse->virtualRegister(), LUse::ANY, + /* usedAtStart = */ true); + } + + if (!vreg(def).addInitialRange(alloc(), from, from.next(), + &numRanges)) { + return false; + } + vreg(def).setInitialDefinition(from); + live.remove(def->virtualRegister()); + } + + for (size_t i = 0; i < ins->numTemps(); i++) { + LDefinition* temp = ins->getTemp(i); + if (temp->isBogusTemp()) { + continue; + } + + // Normally temps are considered to cover both the input + // and output of the associated instruction. In some cases + // though we want to use a fixed register as both an input + // and clobbered register in the instruction, so watch for + // this and shorten the temp to cover only the output. + CodePosition from = inputOf(*ins); + if (temp->policy() == LDefinition::FIXED) { + AnyRegister reg = temp->output()->toRegister(); + for (LInstruction::InputIterator alloc(**ins); alloc.more(); + alloc.next()) { + if (alloc->isUse()) { + LUse* use = alloc->toUse(); + if (use->isFixedRegister()) { + if (GetFixedRegister(vreg(use).def(), use) == reg) { + from = outputOf(*ins); + } + } + } + } + } + + // * For non-call instructions, temps cover both the input and output, + // so temps never alias uses (even at-start uses) or defs. + // * For call instructions, temps only cover the input (the output is + // used for the force-spill ranges added above). This means temps + // still don't alias uses but they can alias the (fixed) defs. For now + // we conservatively require temps to have a fixed register for call + // instructions to prevent a footgun. + MOZ_ASSERT_IF(ins->isCall(), temp->policy() == LDefinition::FIXED); + CodePosition to = + ins->isCall() ? outputOf(*ins) : outputOf(*ins).next(); + + if (!vreg(temp).addInitialRange(alloc(), from, to, &numRanges)) { + return false; + } + vreg(temp).setInitialDefinition(from); + } + + DebugOnly hasUseRegister = false; + DebugOnly hasUseRegisterAtStart = false; + + for (LInstruction::InputIterator inputAlloc(**ins); inputAlloc.more(); + inputAlloc.next()) { + if (inputAlloc->isUse()) { + LUse* use = inputAlloc->toUse(); + + // Call uses should always be at-start, since calls use all + // registers. + MOZ_ASSERT_IF(ins->isCall() && !inputAlloc.isSnapshotInput(), + use->usedAtStart()); + +#ifdef DEBUG + // If there are both useRegisterAtStart(x) and useRegister(y) + // uses, we may assign the same register to both operands + // (bug 772830). Don't allow this for now. + if (use->policy() == LUse::REGISTER) { + if (use->usedAtStart()) { + if (!IsInputReused(*ins, use)) { + hasUseRegisterAtStart = true; + } + } else { + hasUseRegister = true; + } + } + MOZ_ASSERT(!(hasUseRegister && hasUseRegisterAtStart)); +#endif + + // Don't treat RECOVERED_INPUT uses as keeping the vreg alive. + if (use->policy() == LUse::RECOVERED_INPUT) { + continue; + } + + CodePosition to = use->usedAtStart() ? inputOf(*ins) : outputOf(*ins); + if (use->isFixedRegister()) { + LAllocation reg(AnyRegister::FromCode(use->registerCode())); + for (size_t i = 0; i < ins->numDefs(); i++) { + LDefinition* def = ins->getDef(i); + if (def->policy() == LDefinition::FIXED && + *def->output() == reg) { + to = inputOf(*ins); + } + } + } + + if (!vreg(use).addInitialRange(alloc(), entryOf(block), to.next(), + &numRanges)) { + return false; + } + UsePosition* usePosition = + new (alloc().fallible()) UsePosition(use, to); + if (!usePosition) { + return false; + } + vreg(use).addInitialUse(usePosition); + live.insert(use->virtualRegister()); + } + } + } + + // Phis have simultaneous assignment semantics at block begin, so at + // the beginning of the block we can be sure that liveIn does not + // contain any phi outputs. + for (unsigned int i = 0; i < block->numPhis(); i++) { + LDefinition* def = block->getPhi(i)->getDef(0); + if (live.contains(def->virtualRegister())) { + live.remove(def->virtualRegister()); + } else { + // This is a dead phi, so add a dummy range over all phis. This + // can go away if we have an earlier dead code elimination pass. + CodePosition entryPos = entryOf(block); + if (!vreg(def).addInitialRange(alloc(), entryPos, entryPos.next(), + &numRanges)) { + return false; + } + } + } + + if (mblock->isLoopHeader()) { + // A divergence from the published algorithm is required here, as + // our block order does not guarantee that blocks of a loop are + // contiguous. As a result, a single live range spanning the + // loop is not possible. Additionally, we require liveIn in a later + // pass for resolution, so that must also be fixed up here. + MBasicBlock* loopBlock = mblock->backedge(); + while (true) { + // Blocks must already have been visited to have a liveIn set. + MOZ_ASSERT(loopBlock->id() >= mblock->id()); + + // Add a range for this entire loop block + CodePosition from = entryOf(loopBlock->lir()); + CodePosition to = exitOf(loopBlock->lir()).next(); + + for (BitSet::Iterator liveRegId(live); liveRegId; ++liveRegId) { + if (!vregs[*liveRegId].addInitialRange(alloc(), from, to, + &numRanges)) { + return false; + } + } + + // Fix up the liveIn set. + liveIn[loopBlock->id()].insertAll(live); + + // Make sure we don't visit this node again + loopDone.insert(loopBlock->id()); + + // If this is the loop header, any predecessors are either the + // backedge or out of the loop, so skip any predecessors of + // this block + if (loopBlock != mblock) { + for (size_t i = 0; i < loopBlock->numPredecessors(); i++) { + MBasicBlock* pred = loopBlock->getPredecessor(i); + if (loopDone.contains(pred->id())) { + continue; + } + if (!loopWorkList.append(pred)) { + return false; + } + } + } + + // Terminate loop if out of work. + if (loopWorkList.empty()) { + break; + } + + // Grab the next block off the work list, skipping any OSR block. + MBasicBlock* osrBlock = graph.mir().osrBlock(); + while (!loopWorkList.empty()) { + loopBlock = loopWorkList.popCopy(); + if (loopBlock != osrBlock) { + break; + } + } + + // If end is reached without finding a non-OSR block, then no more work + // items were found. + if (loopBlock == osrBlock) { + MOZ_ASSERT(loopWorkList.empty()); + break; + } + } + + // Clear the done set for other loops + loopDone.clear(); + } + + MOZ_ASSERT_IF(!mblock->numPredecessors(), live.empty()); + } + + JitSpew(JitSpew_RegAlloc, "Completed liveness analysis"); + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Merging and queueing of LiveRange groups // +// // +/////////////////////////////////////////////////////////////////////////////// + +// Helper for ::tryMergeBundles +static bool IsArgumentSlotDefinition(LDefinition* def) { + return def->policy() == LDefinition::FIXED && def->output()->isArgument(); +} + +// Helper for ::tryMergeBundles +static bool IsThisSlotDefinition(LDefinition* def) { + return IsArgumentSlotDefinition(def) && + def->output()->toArgument()->index() < + THIS_FRAME_ARGSLOT + sizeof(Value); +} + +// Helper for ::tryMergeBundles +static bool HasStackPolicy(LDefinition* def) { + return def->policy() == LDefinition::STACK; +} + +// Helper for ::tryMergeBundles +static bool CanMergeTypesInBundle(LDefinition::Type a, LDefinition::Type b) { + // Fast path for the common case. + if (a == b) { + return true; + } + + // Only merge if the sizes match, so that we don't get confused about the + // width of spill slots. + return StackSlotAllocator::width(a) == StackSlotAllocator::width(b); +} + +// Helper for ::tryMergeReusedRegister +bool BacktrackingAllocator::tryMergeBundles(LiveBundle* bundle0, + LiveBundle* bundle1) { + // See if bundle0 and bundle1 can be merged together. + if (bundle0 == bundle1) { + return true; + } + + // Get a representative virtual register from each bundle. + VirtualRegister& reg0 = bundle0->firstRange()->vreg(); + VirtualRegister& reg1 = bundle1->firstRange()->vreg(); + + MOZ_ASSERT(CanMergeTypesInBundle(reg0.type(), reg1.type())); + MOZ_ASSERT(reg0.isCompatible(reg1)); + + // Registers which might spill to the frame's |this| slot can only be + // grouped with other such registers. The frame's |this| slot must always + // hold the |this| value, as required by JitFrame tracing and by the Ion + // constructor calling convention. + if (IsThisSlotDefinition(reg0.def()) || IsThisSlotDefinition(reg1.def())) { + if (*reg0.def()->output() != *reg1.def()->output()) { + return true; + } + } + + // Registers which might spill to the frame's argument slots can only be + // grouped with other such registers if the frame might access those + // arguments through a lazy arguments object or rest parameter. + if (IsArgumentSlotDefinition(reg0.def()) || + IsArgumentSlotDefinition(reg1.def())) { + if (graph.mir().entryBlock()->info().mayReadFrameArgsDirectly()) { + if (*reg0.def()->output() != *reg1.def()->output()) { + return true; + } + } + } + + // When we make a call to a WebAssembly function that returns multiple + // results, some of those results can go on the stack. The callee is passed a + // pointer to this stack area, which is represented as having policy + // LDefinition::STACK (with type LDefinition::STACKRESULTS). Individual + // results alias parts of the stack area with a value-appropriate type, but + // policy LDefinition::STACK. This aliasing between allocations makes it + // unsound to merge anything with a LDefinition::STACK policy. + if (HasStackPolicy(reg0.def()) || HasStackPolicy(reg1.def())) { + return true; + } + + // Limit the number of times we compare ranges if there are many ranges in + // one of the bundles, to avoid quadratic behavior. + static const size_t MAX_RANGES = 200; + + // Make sure that ranges in the bundles do not overlap. + LiveRange::BundleLinkIterator iter0 = bundle0->rangesBegin(), + iter1 = bundle1->rangesBegin(); + size_t count = 0; + while (iter0 && iter1) { + if (++count >= MAX_RANGES) { + return true; + } + + LiveRange* range0 = LiveRange::get(*iter0); + LiveRange* range1 = LiveRange::get(*iter1); + + if (range0->from() >= range1->to()) { + iter1++; + } else if (range1->from() >= range0->to()) { + iter0++; + } else { + return true; + } + } + + // Move all ranges from bundle1 into bundle0. + while (LiveRange* range = bundle1->popFirstRange()) { + bundle0->addRange(range); + } + + return true; +} + +// Helper for ::mergeAndQueueRegisters +void BacktrackingAllocator::allocateStackDefinition(VirtualRegister& reg) { + LInstruction* ins = reg.ins()->toInstruction(); + if (reg.def()->type() == LDefinition::STACKRESULTS) { + LStackArea alloc(ins->toInstruction()); + stackSlotAllocator.allocateStackArea(&alloc); + reg.def()->setOutput(alloc); + } else { + // Because the definitions are visited in order, the area has been allocated + // before we reach this result, so we know the operand is an LStackArea. + const LUse* use = ins->getOperand(0)->toUse(); + VirtualRegister& area = vregs[use->virtualRegister()]; + const LStackArea* areaAlloc = area.def()->output()->toStackArea(); + reg.def()->setOutput(areaAlloc->resultAlloc(ins, reg.def())); + } +} + +// Helper for ::mergeAndQueueRegisters +bool BacktrackingAllocator::tryMergeReusedRegister(VirtualRegister& def, + VirtualRegister& input) { + // def is a vreg which reuses input for its output physical register. Try + // to merge ranges for def with those of input if possible, as avoiding + // copies before def's instruction is crucial for generated code quality + // (MUST_REUSE_INPUT is used for all arithmetic on x86/x64). + + if (def.rangeFor(inputOf(def.ins()))) { + MOZ_ASSERT(def.isTemp()); + def.setMustCopyInput(); + return true; + } + + if (!CanMergeTypesInBundle(def.type(), input.type())) { + def.setMustCopyInput(); + return true; + } + + LiveRange* inputRange = input.rangeFor(outputOf(def.ins())); + if (!inputRange) { + // The input is not live after the instruction, either in a safepoint + // for the instruction or in subsequent code. The input and output + // can thus be in the same group. + return tryMergeBundles(def.firstBundle(), input.firstBundle()); + } + + // Avoid merging in very large live ranges as merging has non-linear + // complexity. The cutoff value is hard to gauge. 1M was chosen because it + // is "large" and yet usefully caps compile time on AutoCad-for-the-web to + // something reasonable on a 2017-era desktop system. + const uint32_t RANGE_SIZE_CUTOFF = 1000000; + if (inputRange->to() - inputRange->from() > RANGE_SIZE_CUTOFF) { + def.setMustCopyInput(); + return true; + } + + // The input is live afterwards, either in future instructions or in a + // safepoint for the reusing instruction. This is impossible to satisfy + // without copying the input. + // + // It may or may not be better to split the input into two bundles at the + // point of the definition, which may permit merging. One case where it is + // definitely better to split is if the input never has any register uses + // after the instruction. Handle this splitting eagerly. + + LBlock* block = def.ins()->block(); + + // The input's lifetime must end within the same block as the definition, + // otherwise it could live on in phis elsewhere. + if (inputRange != input.lastRange() || inputRange->to() > exitOf(block)) { + def.setMustCopyInput(); + return true; + } + + // If we already split the input for some other register, don't make a + // third bundle. + if (inputRange->bundle() != input.firstRange()->bundle()) { + def.setMustCopyInput(); + return true; + } + + // If the input will start out in memory then adding a separate bundle for + // memory uses after the def won't help. + if (input.def()->isFixed() && !input.def()->output()->isRegister()) { + def.setMustCopyInput(); + return true; + } + + // The input cannot have register or reused uses after the definition. + for (UsePositionIterator iter = inputRange->usesBegin(); iter; iter++) { + if (iter->pos <= inputOf(def.ins())) { + continue; + } + + LUse* use = iter->use(); + if (FindReusingDefOrTemp(insData[iter->pos], use)) { + def.setMustCopyInput(); + return true; + } + if (iter->usePolicy() != LUse::ANY && + iter->usePolicy() != LUse::KEEPALIVE) { + def.setMustCopyInput(); + return true; + } + } + + LiveRange* preRange = LiveRange::FallibleNew( + alloc(), &input, inputRange->from(), outputOf(def.ins())); + if (!preRange) { + return false; + } + + // The new range starts at reg's input position, which means it overlaps + // with the old range at one position. This is what we want, because we + // need to copy the input before the instruction. + LiveRange* postRange = LiveRange::FallibleNew( + alloc(), &input, inputOf(def.ins()), inputRange->to()); + if (!postRange) { + return false; + } + + inputRange->tryToMoveDefAndUsesInto(preRange); + inputRange->tryToMoveDefAndUsesInto(postRange); + MOZ_ASSERT(!inputRange->hasUses()); + + JitSpewIfEnabled(JitSpew_RegAlloc, + " splitting reused input at %u to try to help grouping", + inputOf(def.ins()).bits()); + + LiveBundle* firstBundle = inputRange->bundle(); + input.removeRange(inputRange); + input.addRange(preRange); + input.addRange(postRange); + + firstBundle->removeRange(inputRange); + firstBundle->addRange(preRange); + + // The new range goes in a separate bundle, where it will be spilled during + // allocation. + LiveBundle* secondBundle = LiveBundle::FallibleNew(alloc(), nullptr, nullptr); + if (!secondBundle) { + return false; + } + secondBundle->addRange(postRange); + + return tryMergeBundles(def.firstBundle(), input.firstBundle()); +} + +bool BacktrackingAllocator::mergeAndQueueRegisters() { + MOZ_ASSERT(!vregs[0u].hasRanges()); + + // Create a bundle for each register containing all its ranges. + for (size_t i = 1; i < graph.numVirtualRegisters(); i++) { + VirtualRegister& reg = vregs[i]; + if (!reg.hasRanges()) { + continue; + } + + LiveBundle* bundle = LiveBundle::FallibleNew(alloc(), nullptr, nullptr); + if (!bundle) { + return false; + } + for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + bundle->addRange(range); + } + } + + // If there is an OSR block, merge parameters in that block with the + // corresponding parameters in the initial block. + if (MBasicBlock* osr = graph.mir().osrBlock()) { + size_t original = 1; + for (LInstructionIterator iter = osr->lir()->begin(); + iter != osr->lir()->end(); iter++) { + if (iter->isParameter()) { + for (size_t i = 0; i < iter->numDefs(); i++) { + DebugOnly found = false; + VirtualRegister& paramVreg = vreg(iter->getDef(i)); + for (; original < paramVreg.vreg(); original++) { + VirtualRegister& originalVreg = vregs[original]; + if (*originalVreg.def()->output() == *iter->getDef(i)->output()) { + MOZ_ASSERT(originalVreg.ins()->isParameter()); + if (!tryMergeBundles(originalVreg.firstBundle(), + paramVreg.firstBundle())) { + return false; + } + found = true; + break; + } + } + MOZ_ASSERT(found); + } + } + } + } + + // Try to merge registers with their reused inputs. + for (size_t i = 1; i < graph.numVirtualRegisters(); i++) { + VirtualRegister& reg = vregs[i]; + if (!reg.hasRanges()) { + continue; + } + + if (reg.def()->policy() == LDefinition::MUST_REUSE_INPUT) { + LUse* use = reg.ins() + ->toInstruction() + ->getOperand(reg.def()->getReusedInput()) + ->toUse(); + if (!tryMergeReusedRegister(reg, vreg(use))) { + return false; + } + } + } + + // Try to merge phis with their inputs. + for (size_t i = 0; i < graph.numBlocks(); i++) { + LBlock* block = graph.getBlock(i); + for (size_t j = 0; j < block->numPhis(); j++) { + LPhi* phi = block->getPhi(j); + VirtualRegister& outputVreg = vreg(phi->getDef(0)); + for (size_t k = 0, kend = phi->numOperands(); k < kend; k++) { + VirtualRegister& inputVreg = vreg(phi->getOperand(k)->toUse()); + if (!tryMergeBundles(inputVreg.firstBundle(), + outputVreg.firstBundle())) { + return false; + } + } + } + } + + // Add all bundles to the allocation queue, and create spill sets for them. + for (size_t i = 1; i < graph.numVirtualRegisters(); i++) { + VirtualRegister& reg = vregs[i]; + + // Eagerly allocate stack result areas and their component stack results. + if (reg.def() && reg.def()->policy() == LDefinition::STACK) { + allocateStackDefinition(reg); + } + + for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + LiveBundle* bundle = range->bundle(); + if (range == bundle->firstRange()) { + if (!alloc().ensureBallast()) { + return false; + } + + SpillSet* spill = SpillSet::New(alloc()); + if (!spill) { + return false; + } + bundle->setSpillSet(spill); + + size_t priority = computePriority(bundle); + if (!allocationQueue.insert(QueueItem(bundle, priority))) { + return false; + } + } + } + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Code for the splitting/spilling subsystem begins here. // +// // +// The code that follows is structured in the following sequence: // +// // +// (1) Routines that are helpers for ::splitAt. // +// (2) ::splitAt itself, which implements splitting decisions. // +// (3) heuristic routines (eg ::splitAcrossCalls), which decide where // +// splits should be made. They then call ::splitAt to perform the // +// chosen split. // +// (4) The top level driver, ::chooseBundleSplit. // +// // +// There are further comments on ::splitAt and ::chooseBundleSplit below. // +// // +/////////////////////////////////////////////////////////////////////////////// + +/////////////////////////////////////////////////////////////////////////////// +// // +// Implementation of splitting decisions, but not the making of those // +// decisions: various helper functions // +// // +/////////////////////////////////////////////////////////////////////////////// + +bool BacktrackingAllocator::updateVirtualRegisterListsThenRequeueBundles( + LiveBundle* bundle, const LiveBundleVector& newBundles) { +#ifdef DEBUG + if (newBundles.length() == 1) { + LiveBundle* newBundle = newBundles[0]; + if (newBundle->numRanges() == bundle->numRanges() && + computePriority(newBundle) == computePriority(bundle)) { + bool different = false; + LiveRange::BundleLinkIterator oldRanges = bundle->rangesBegin(); + LiveRange::BundleLinkIterator newRanges = newBundle->rangesBegin(); + while (oldRanges) { + LiveRange* oldRange = LiveRange::get(*oldRanges); + LiveRange* newRange = LiveRange::get(*newRanges); + if (oldRange->from() != newRange->from() || + oldRange->to() != newRange->to()) { + different = true; + break; + } + oldRanges++; + newRanges++; + } + + // This is likely to trigger an infinite loop in register allocation. This + // can be the result of invalid register constraints, making regalloc + // impossible; consider relaxing those. + MOZ_ASSERT(different, + "Split results in the same bundle with the same priority"); + } + } +#endif + + if (JitSpewEnabled(JitSpew_RegAlloc)) { + JitSpew(JitSpew_RegAlloc, " .. into:"); + for (size_t i = 0; i < newBundles.length(); i++) { + JitSpew(JitSpew_RegAlloc, " %s", newBundles[i]->toString().get()); + } + } + + // Remove all ranges in the old bundle from their register's list. + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + range->vreg().removeRange(range); + } + + // Add all ranges in the new bundles to their register's list. + for (size_t i = 0; i < newBundles.length(); i++) { + LiveBundle* newBundle = newBundles[i]; + for (LiveRange::BundleLinkIterator iter = newBundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + range->vreg().addRange(range); + } + } + + // Queue the new bundles for register assignment. + for (size_t i = 0; i < newBundles.length(); i++) { + LiveBundle* newBundle = newBundles[i]; + size_t priority = computePriority(newBundle); + if (!allocationQueue.insert(QueueItem(newBundle, priority))) { + return false; + } + } + + return true; +} + +// Helper for ::splitAt +// When splitting a bundle according to a list of split positions, return +// whether a use or range at |pos| should use a different bundle than the last +// position this was called for. +static bool UseNewBundle(const SplitPositionVector& splitPositions, + CodePosition pos, size_t* activeSplitPosition) { + if (splitPositions.empty()) { + // When the split positions are empty we are splitting at all uses. + return true; + } + + if (*activeSplitPosition == splitPositions.length()) { + // We've advanced past all split positions. + return false; + } + + if (splitPositions[*activeSplitPosition] > pos) { + // We haven't gotten to the next split position yet. + return false; + } + + // We've advanced past the next split position, find the next one which we + // should split at. + while (*activeSplitPosition < splitPositions.length() && + splitPositions[*activeSplitPosition] <= pos) { + (*activeSplitPosition)++; + } + return true; +} + +// Helper for ::splitAt +static bool HasPrecedingRangeSharingVreg(LiveBundle* bundle, LiveRange* range) { + MOZ_ASSERT(range->bundle() == bundle); + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* prevRange = LiveRange::get(*iter); + if (prevRange == range) { + return false; + } + if (&prevRange->vreg() == &range->vreg()) { + return true; + } + } + + MOZ_CRASH(); +} + +// Helper for ::splitAt +static bool HasFollowingRangeSharingVreg(LiveBundle* bundle, LiveRange* range) { + MOZ_ASSERT(range->bundle() == bundle); + + bool foundRange = false; + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* prevRange = LiveRange::get(*iter); + if (foundRange && &prevRange->vreg() == &range->vreg()) { + return true; + } + if (prevRange == range) { + foundRange = true; + } + } + + MOZ_ASSERT(foundRange); + return false; +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Implementation of splitting decisions, but not the making of those // +// decisions: // +// ::splitAt // +// // +/////////////////////////////////////////////////////////////////////////////// + +// ::splitAt +// --------- +// It would be nice to be able to interpret ::splitAt as simply performing +// whatever split the heuristic routines decide on. Unfortunately it +// tries to "improve" on the initial locations, which as +// https://bugzilla.mozilla.org/show_bug.cgi?id=1758274#c17 shows, often +// leads to excessive spilling. So there is no clean distinction between +// policy (where to split, computed by the heuristic routines) and +// implementation (done by ::splitAt). +// +// ::splitAt -- creation of spill parent bundles +// --------------------------------------------- +// To understand what ::splitAt does, we must refer back to Section 1's +// description of LiveBundle::spillParent_. +// +// Initially (as created by Phase 1), all bundles have `spillParent_` being +// NULL. If ::splitAt is asked to split such a bundle, it will first create a +// "spill bundle" or "spill parent" bundle. This is a copy of the original, +// with two changes: +// +// * all register uses have been removed, so that only stack-compatible uses +// remain. +// +// * for all LiveRanges in the bundle that define a register, the start point +// of the range is moved one CodePosition forwards, thusly: +// +// from = minimalDefEnd(insData[from]).next(); +// +// The reason for the latter relates to the idea described in Section 1, that +// all LiveRanges for any given VirtualRegister must form a tree rooted at the +// defining LiveRange. If the spill-bundle definition range start points are +// the same as those in the original bundle, then we will end up with two roots +// for the tree, and it is then unclear which one should supply "the value". +// +// Putting the spill-bundle start point one CodePosition further along causes +// the range containing the register def (after splitting) to still be the +// defining point. ::createMoveGroupsFromLiveRangeTransitions will observe the +// equivalent spill-bundle range starting one point later and add a MoveGroup +// to move the value into it. Since the spill bundle is intended to be stack +// resident, the effect is to force creation of the MoveGroup that will +// actually spill this value onto the stack. +// +// If the bundle provided to ::splitAt already has a spill parent, then +// ::splitAt doesn't create a new spill parent. This situation will happen +// when the bundle to be split was itself created by splitting. The effect is +// that *all* bundles created from an "original bundle" share the same spill +// parent, and hence they will share the same spill slot, which guarantees that +// all the spilled fragments of a VirtualRegister share the same spill slot, +// which means we'll never have to move a VirtualRegister between different +// spill slots during its lifetime. +// +// ::splitAt -- creation of other bundles +// -------------------------------------- +// With the spill parent bundle question out of the way, ::splitAt then goes on +// to create the remaining new bundles, using near-incomprehensible logic +// steered by `UseNewBundle`. +// +// This supposedly splits the bundle at the positions given by the +// `SplitPositionVector` parameter to ::splitAt, putting them in a temporary +// vector `newBundles`. Whether it really splits at the requested positions or +// not is hard to say; more important is what happens next. +// +// ::splitAt -- "improvement" ("filtering") of the split bundles +// ------------------------------------------------------------- +// ::splitAt now tries to reduce the length of the LiveRanges that make up the +// new bundles (not including the "spill parent"). I assume this is to remove +// sections of the bundles that carry no useful value (eg, extending after the +// last using a range), thereby removing the demand for registers in those +// parts. This does however mean that ::splitAt is no longer really splitting +// where the heuristic routines wanted, and that can lead to a big increase in +// spilling in loops, as +// https://bugzilla.mozilla.org/show_bug.cgi?id=1758274#c17 describes. +// +// ::splitAt -- meaning of the incoming `SplitPositionVector` +// ---------------------------------------------------------- +// ::splitAt has one last mystery which is important to document. The split +// positions are specified as CodePositions, but this leads to ambiguity +// because, in a sequence of N (LIR) instructions, there are 2N valid +// CodePositions. For example: +// +// 6-7 WasmLoadTls [def v2] [use v1:R] +// 8-9 WasmNullConstant [def v3] +// +// Consider splitting the range for `v2`, which starts at CodePosition 7. +// What's the difference between saying "split it at 7" and "split it at 8" ? +// Not much really, since in both cases what we intend is for the range to be +// split in between the two instructions. +// +// Hence I believe the semantics is: +// +// * splitting at an even numbered CodePosition (eg, 8), which is an input-side +// position, means "split before the instruction containing this position". +// +// * splitting at an odd numbered CodePositin (eg, 7), which is an output-side +// position, means "split after the instruction containing this position". +// +// Hence in the example, we could specify either 7 or 8 to mean the same +// placement of the split. Well, almost true, but actually: +// +// (SPEC) specifying 8 means +// +// "split between these two insns, and any resulting MoveGroup goes in the +// list to be emitted before the start of the second insn" +// +// (SPEC) specifying 7 means +// +// "split between these two insns, and any resulting MoveGroup goes in the +// list to be emitted after the end of the first insn" +// +// In most cases we don't care on which "side of the valley" the MoveGroup ends +// up, in which case we can use either convention. +// +// (SPEC) I believe these semantics are implied by the logic in +// ::createMoveGroupsFromLiveRangeTransitions. They are certainly not +// documented anywhere in the code. + +bool BacktrackingAllocator::splitAt(LiveBundle* bundle, + const SplitPositionVector& splitPositions) { + // Split the bundle at the given split points. Register uses which have no + // intervening split points are consolidated into the same bundle. If the + // list of split points is empty, then all register uses are placed in + // minimal bundles. + + // splitPositions should be sorted. + for (size_t i = 1; i < splitPositions.length(); ++i) { + MOZ_ASSERT(splitPositions[i - 1] < splitPositions[i]); + } + + // We don't need to create a new spill bundle if there already is one. + bool spillBundleIsNew = false; + LiveBundle* spillBundle = bundle->spillParent(); + if (!spillBundle) { + spillBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(), nullptr); + if (!spillBundle) { + return false; + } + spillBundleIsNew = true; + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + + CodePosition from = range->from(); + if (isRegisterDefinition(range)) { + from = minimalDefEnd(insData[from]).next(); + } + + if (from < range->to()) { + if (!spillBundle->addRange(alloc(), &range->vreg(), from, + range->to())) { + return false; + } + + if (range->hasDefinition() && !isRegisterDefinition(range)) { + spillBundle->lastRange()->setHasDefinition(); + } + } + } + } + + LiveBundleVector newBundles; + + // The bundle which ranges are currently being added to. + LiveBundle* activeBundle = + LiveBundle::FallibleNew(alloc(), bundle->spillSet(), spillBundle); + if (!activeBundle || !newBundles.append(activeBundle)) { + return false; + } + + // State for use by UseNewBundle. + size_t activeSplitPosition = 0; + + // Make new bundles according to the split positions, and distribute ranges + // and uses to them. + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + + if (UseNewBundle(splitPositions, range->from(), &activeSplitPosition)) { + activeBundle = + LiveBundle::FallibleNew(alloc(), bundle->spillSet(), spillBundle); + if (!activeBundle || !newBundles.append(activeBundle)) { + return false; + } + } + + LiveRange* activeRange = LiveRange::FallibleNew(alloc(), &range->vreg(), + range->from(), range->to()); + if (!activeRange) { + return false; + } + activeBundle->addRange(activeRange); + + if (isRegisterDefinition(range)) { + activeRange->setHasDefinition(); + } + + while (range->hasUses()) { + UsePosition* use = range->popUse(); + LNode* ins = insData[use->pos]; + + // Any uses of a register that appear before its definition has + // finished must be associated with the range for that definition. + if (isRegisterDefinition(range) && + use->pos <= minimalDefEnd(insData[range->from()])) { + activeRange->addUse(use); + } else if (isRegisterUse(use, ins)) { + // Place this register use into a different bundle from the + // last one if there are any split points between the two uses. + // UseNewBundle always returns true if we are splitting at all + // register uses, but we can still reuse the last range and + // bundle if they have uses at the same position, except when + // either use is fixed (the two uses might require incompatible + // registers.) + if (UseNewBundle(splitPositions, use->pos, &activeSplitPosition) && + (!activeRange->hasUses() || + activeRange->usesBegin()->pos != use->pos || + activeRange->usesBegin()->usePolicy() == LUse::FIXED || + use->usePolicy() == LUse::FIXED)) { + activeBundle = + LiveBundle::FallibleNew(alloc(), bundle->spillSet(), spillBundle); + if (!activeBundle || !newBundles.append(activeBundle)) { + return false; + } + activeRange = LiveRange::FallibleNew(alloc(), &range->vreg(), + range->from(), range->to()); + if (!activeRange) { + return false; + } + activeBundle->addRange(activeRange); + } + + activeRange->addUse(use); + } else { + MOZ_ASSERT(spillBundleIsNew); + spillBundle->rangeFor(use->pos)->addUse(use); + } + } + } + + LiveBundleVector filteredBundles; + + // Trim the ends of ranges in each new bundle when there are no other + // earlier or later ranges in the same bundle with the same vreg. + for (size_t i = 0; i < newBundles.length(); i++) { + LiveBundle* bundle = newBundles[i]; + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;) { + LiveRange* range = LiveRange::get(*iter); + + if (!range->hasDefinition()) { + if (!HasPrecedingRangeSharingVreg(bundle, range)) { + if (range->hasUses()) { + UsePosition* use = *range->usesBegin(); + range->setFrom(inputOf(insData[use->pos])); + } else { + bundle->removeRangeAndIncrementIterator(iter); + continue; + } + } + } + + if (!HasFollowingRangeSharingVreg(bundle, range)) { + if (range->hasUses()) { + UsePosition* use = range->lastUse(); + range->setTo(use->pos.next()); + } else if (range->hasDefinition()) { + range->setTo(minimalDefEnd(insData[range->from()]).next()); + } else { + bundle->removeRangeAndIncrementIterator(iter); + continue; + } + } + + iter++; + } + + if (bundle->hasRanges() && !filteredBundles.append(bundle)) { + return false; + } + } + + if (spillBundleIsNew && !filteredBundles.append(spillBundle)) { + return false; + } + + return updateVirtualRegisterListsThenRequeueBundles(bundle, filteredBundles); +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Creation of splitting decisions, but not their implementation: // +// ::splitAcrossCalls // +// ::trySplitAcrossHotcode // +// ::trySplitAfterLastRegisterUse // +// ::trySplitBeforeFirstRegisterUse // +// // +/////////////////////////////////////////////////////////////////////////////// + +bool BacktrackingAllocator::splitAcrossCalls(LiveBundle* bundle) { + // Split the bundle to separate register uses and non-register uses and + // allow the vreg to be spilled across its range. + + // Find the locations of all calls in the bundle's range. + SplitPositionVector callPositions; + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + CallRange searchRange(range->from(), range->to()); + CallRange* callRange; + if (!callRanges.contains(&searchRange, &callRange)) { + // There are no calls inside this range. + continue; + } + MOZ_ASSERT(range->covers(callRange->range.from)); + + // The search above returns an arbitrary call within the range. Walk + // backwards to find the first call in the range. + for (CallRangeList::reverse_iterator riter = + callRangesList.rbegin(callRange); + riter != callRangesList.rend(); ++riter) { + CodePosition pos = riter->range.from; + if (range->covers(pos)) { + callRange = *riter; + } else { + break; + } + } + + // Add all call positions within the range, by walking forwards. + for (CallRangeList::iterator iter = callRangesList.begin(callRange); + iter != callRangesList.end(); ++iter) { + CodePosition pos = iter->range.from; + if (!range->covers(pos)) { + break; + } + + // Calls at the beginning of the range are ignored; there is no splitting + // to do. + if (range->covers(pos.previous())) { + MOZ_ASSERT_IF(callPositions.length(), pos > callPositions.back()); + if (!callPositions.append(pos)) { + return false; + } + } + } + } + MOZ_ASSERT(callPositions.length()); + +#ifdef JS_JITSPEW + JitSpewStart(JitSpew_RegAlloc, " .. split across calls at "); + for (size_t i = 0; i < callPositions.length(); ++i) { + JitSpewCont(JitSpew_RegAlloc, "%s%u", i != 0 ? ", " : "", + callPositions[i].bits()); + } + JitSpewFin(JitSpew_RegAlloc); +#endif + + return splitAt(bundle, callPositions); +} + +bool BacktrackingAllocator::trySplitAcrossHotcode(LiveBundle* bundle, + bool* success) { + // If this bundle has portions that are hot and portions that are cold, + // split it at the boundaries between hot and cold code. + + LiveRange* hotRange = nullptr; + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + if (hotcode.contains(range, &hotRange)) { + break; + } + } + + // Don't split if there is no hot code in the bundle. + if (!hotRange) { + JitSpew(JitSpew_RegAlloc, " .. bundle does not contain hot code"); + return true; + } + + // Don't split if there is no cold code in the bundle. + bool coldCode = false; + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + if (!hotRange->contains(range)) { + coldCode = true; + break; + } + } + if (!coldCode) { + JitSpew(JitSpew_RegAlloc, " .. bundle does not contain cold code"); + return true; + } + + JitSpewIfEnabled(JitSpew_RegAlloc, " .. split across hot range %s", + hotRange->toString().get()); + + // Tweak the splitting method when compiling wasm code to look at actual + // uses within the hot/cold code. This heuristic is in place as the below + // mechanism regresses several asm.js tests. Hopefully this will be fixed + // soon and this special case removed. See bug 948838. + if (compilingWasm()) { + SplitPositionVector splitPositions; + if (!splitPositions.append(hotRange->from()) || + !splitPositions.append(hotRange->to())) { + return false; + } + *success = true; + return splitAt(bundle, splitPositions); + } + + LiveBundle* hotBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(), + bundle->spillParent()); + if (!hotBundle) { + return false; + } + LiveBundle* preBundle = nullptr; + LiveBundle* postBundle = nullptr; + LiveBundle* coldBundle = nullptr; + + if (testbed) { + coldBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(), + bundle->spillParent()); + if (!coldBundle) { + return false; + } + } + + // Accumulate the ranges of hot and cold code in the bundle. Note that + // we are only comparing with the single hot range found, so the cold code + // may contain separate hot ranges. + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + LiveRange::Range hot, coldPre, coldPost; + range->intersect(hotRange, &coldPre, &hot, &coldPost); + + if (!hot.empty()) { + if (!hotBundle->addRangeAndDistributeUses(alloc(), range, hot.from, + hot.to)) { + return false; + } + } + + if (!coldPre.empty()) { + if (testbed) { + if (!coldBundle->addRangeAndDistributeUses(alloc(), range, coldPre.from, + coldPre.to)) { + return false; + } + } else { + if (!preBundle) { + preBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(), + bundle->spillParent()); + if (!preBundle) { + return false; + } + } + if (!preBundle->addRangeAndDistributeUses(alloc(), range, coldPre.from, + coldPre.to)) { + return false; + } + } + } + + if (!coldPost.empty()) { + if (testbed) { + if (!coldBundle->addRangeAndDistributeUses( + alloc(), range, coldPost.from, coldPost.to)) { + return false; + } + } else { + if (!postBundle) { + postBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(), + bundle->spillParent()); + if (!postBundle) { + return false; + } + } + if (!postBundle->addRangeAndDistributeUses( + alloc(), range, coldPost.from, coldPost.to)) { + return false; + } + } + } + } + + MOZ_ASSERT(hotBundle->numRanges() != 0); + + LiveBundleVector newBundles; + if (!newBundles.append(hotBundle)) { + return false; + } + + if (testbed) { + MOZ_ASSERT(coldBundle->numRanges() != 0); + if (!newBundles.append(coldBundle)) { + return false; + } + } else { + MOZ_ASSERT(preBundle || postBundle); + if (preBundle && !newBundles.append(preBundle)) { + return false; + } + if (postBundle && !newBundles.append(postBundle)) { + return false; + } + } + + *success = true; + return updateVirtualRegisterListsThenRequeueBundles(bundle, newBundles); +} + +bool BacktrackingAllocator::trySplitAfterLastRegisterUse(LiveBundle* bundle, + LiveBundle* conflict, + bool* success) { + // If this bundle's later uses do not require it to be in a register, + // split it after the last use which does require a register. If conflict + // is specified, only consider register uses before the conflict starts. + + CodePosition lastRegisterFrom, lastRegisterTo, lastUse; + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + + // If the range defines a register, consider that a register use for + // our purposes here. + if (isRegisterDefinition(range)) { + CodePosition spillStart = minimalDefEnd(insData[range->from()]).next(); + if (!conflict || spillStart < conflict->firstRange()->from()) { + lastUse = lastRegisterFrom = range->from(); + lastRegisterTo = spillStart; + } + } + + for (UsePositionIterator iter(range->usesBegin()); iter; iter++) { + LNode* ins = insData[iter->pos]; + + // Uses in the bundle should be sorted. + MOZ_ASSERT(iter->pos >= lastUse); + lastUse = inputOf(ins); + + if (!conflict || outputOf(ins) < conflict->firstRange()->from()) { + if (isRegisterUse(*iter, ins, /* considerCopy = */ true)) { + lastRegisterFrom = inputOf(ins); + lastRegisterTo = iter->pos.next(); + } + } + } + } + + // Can't trim non-register uses off the end by splitting. + if (!lastRegisterFrom.bits()) { + JitSpew(JitSpew_RegAlloc, " .. bundle has no register uses"); + return true; + } + if (lastUse < lastRegisterTo) { + JitSpew(JitSpew_RegAlloc, " .. bundle's last use is a register use"); + return true; + } + + JitSpewIfEnabled(JitSpew_RegAlloc, " .. split after last register use at %u", + lastRegisterTo.bits()); + + SplitPositionVector splitPositions; + if (!splitPositions.append(lastRegisterTo)) { + return false; + } + *success = true; + return splitAt(bundle, splitPositions); +} + +bool BacktrackingAllocator::trySplitBeforeFirstRegisterUse(LiveBundle* bundle, + LiveBundle* conflict, + bool* success) { + // If this bundle's earlier uses do not require it to be in a register, + // split it before the first use which does require a register. If conflict + // is specified, only consider register uses after the conflict ends. + + if (isRegisterDefinition(bundle->firstRange())) { + JitSpew(JitSpew_RegAlloc, " .. bundle is defined by a register"); + return true; + } + if (!bundle->firstRange()->hasDefinition()) { + JitSpew(JitSpew_RegAlloc, " .. bundle does not have definition"); + return true; + } + + CodePosition firstRegisterFrom; + + CodePosition conflictEnd; + if (conflict) { + for (LiveRange::BundleLinkIterator iter = conflict->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + if (range->to() > conflictEnd) { + conflictEnd = range->to(); + } + } + } + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + + if (!conflict || range->from() > conflictEnd) { + if (range->hasDefinition() && isRegisterDefinition(range)) { + firstRegisterFrom = range->from(); + break; + } + } + + for (UsePositionIterator iter(range->usesBegin()); iter; iter++) { + LNode* ins = insData[iter->pos]; + + if (!conflict || outputOf(ins) >= conflictEnd) { + if (isRegisterUse(*iter, ins, /* considerCopy = */ true)) { + firstRegisterFrom = inputOf(ins); + break; + } + } + } + if (firstRegisterFrom.bits()) { + break; + } + } + + if (!firstRegisterFrom.bits()) { + // Can't trim non-register uses off the beginning by splitting. + JitSpew(JitSpew_RegAlloc, " bundle has no register uses"); + return true; + } + + JitSpewIfEnabled(JitSpew_RegAlloc, + " .. split before first register use at %u", + firstRegisterFrom.bits()); + + SplitPositionVector splitPositions; + if (!splitPositions.append(firstRegisterFrom)) { + return false; + } + *success = true; + return splitAt(bundle, splitPositions); +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// The top level driver for the splitting machinery // +// // +/////////////////////////////////////////////////////////////////////////////// + +// ::chooseBundleSplit +// ------------------- +// If the first allocation loop (in ::go) can't allocate a bundle, it hands it +// off to ::chooseBundleSplit, which is the "entry point" of the bundle-split +// machinery. This tries four heuristics in turn, to see if any can split the +// bundle: +// +// * ::trySplitAcrossHotcode +// * ::splitAcrossCalls (in some cases) +// * ::trySplitBeforeFirstRegisterUse +// * ::trySplitAfterLastRegisterUse +// +// These routines have similar structure: they try to decide on one or more +// CodePositions at which to split the bundle, using whatever heuristics they +// have to hand. If suitable CodePosition(s) are found, they are put into a +// `SplitPositionVector`, and the bundle and the vector are handed off to +// ::splitAt, which performs the split (at least in theory) at the chosen +// positions. It also arranges for the new bundles to be added to +// ::allocationQueue. +// +// ::trySplitAcrossHotcode has a special case for JS -- it modifies the +// bundle(s) itself, rather than using ::splitAt. +// +// If none of the heuristic routines apply, then ::splitAt is called with an +// empty vector of split points. This is interpreted to mean "split at all +// register uses". When combined with how ::splitAt works, the effect is to +// spill the bundle. + +bool BacktrackingAllocator::chooseBundleSplit(LiveBundle* bundle, bool fixed, + LiveBundle* conflict) { + bool success = false; + + JitSpewIfEnabled(JitSpew_RegAlloc, " Splitting %s ..", + bundle->toString().get()); + + if (!trySplitAcrossHotcode(bundle, &success)) { + return false; + } + if (success) { + return true; + } + + if (fixed) { + return splitAcrossCalls(bundle); + } + + if (!trySplitBeforeFirstRegisterUse(bundle, conflict, &success)) { + return false; + } + if (success) { + return true; + } + + if (!trySplitAfterLastRegisterUse(bundle, conflict, &success)) { + return false; + } + if (success) { + return true; + } + + // Split at all register uses. + SplitPositionVector emptyPositions; + return splitAt(bundle, emptyPositions); +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Bundle allocation // +// // +/////////////////////////////////////////////////////////////////////////////// + +static const size_t MAX_ATTEMPTS = 2; + +bool BacktrackingAllocator::computeRequirement(LiveBundle* bundle, + Requirement* requirement, + Requirement* hint) { + // Set any requirement or hint on bundle according to its definition and + // uses. Return false if there are conflicting requirements which will + // require the bundle to be split. + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + VirtualRegister& reg = range->vreg(); + + if (range->hasDefinition()) { + // Deal with any definition constraints/hints. + LDefinition::Policy policy = reg.def()->policy(); + if (policy == LDefinition::FIXED || policy == LDefinition::STACK) { + // Fixed and stack policies get a FIXED requirement. (In the stack + // case, the allocation should have been performed already by + // mergeAndQueueRegisters.) + JitSpewIfEnabled(JitSpew_RegAlloc, + " Requirement %s, fixed by definition", + reg.def()->output()->toString().get()); + if (!requirement->merge(Requirement(*reg.def()->output()))) { + return false; + } + } else if (reg.ins()->isPhi()) { + // Phis don't have any requirements, but they should prefer their + // input allocations. This is captured by the group hints above. + } else { + // Non-phis get a REGISTER requirement. + if (!requirement->merge(Requirement(Requirement::REGISTER))) { + return false; + } + } + } + + // Search uses for requirements. + for (UsePositionIterator iter = range->usesBegin(); iter; iter++) { + LUse::Policy policy = iter->usePolicy(); + if (policy == LUse::FIXED) { + AnyRegister required = GetFixedRegister(reg.def(), iter->use()); + + JitSpewIfEnabled(JitSpew_RegAlloc, " Requirement %s, due to use at %u", + required.name(), iter->pos.bits()); + + // If there are multiple fixed registers which the bundle is + // required to use, fail. The bundle will need to be split before + // it can be allocated. + if (!requirement->merge(Requirement(LAllocation(required)))) { + return false; + } + } else if (policy == LUse::REGISTER) { + if (!requirement->merge(Requirement(Requirement::REGISTER))) { + return false; + } + } else if (policy == LUse::ANY) { + // ANY differs from KEEPALIVE by actively preferring a register. + if (!hint->merge(Requirement(Requirement::REGISTER))) { + return false; + } + } + + // The only case of STACK use policies is individual stack results using + // their containing stack result area, which is given a fixed allocation + // above. + MOZ_ASSERT_IF(policy == LUse::STACK, + requirement->kind() == Requirement::FIXED); + MOZ_ASSERT_IF(policy == LUse::STACK, + requirement->allocation().isStackArea()); + } + } + + return true; +} + +bool BacktrackingAllocator::tryAllocateRegister(PhysicalRegister& r, + LiveBundle* bundle, + bool* success, bool* pfixed, + LiveBundleVector& conflicting) { + *success = false; + + if (!r.allocatable) { + return true; + } + + LiveBundleVector aliasedConflicting; + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + LiveRangePlus rangePlus(range); + + // All ranges in the bundle must be compatible with the physical register. + MOZ_ASSERT(range->vreg().isCompatible(r.reg)); + + for (size_t a = 0; a < r.reg.numAliased(); a++) { + PhysicalRegister& rAlias = registers[r.reg.aliased(a).code()]; + LiveRangePlus existingPlus; + if (!rAlias.allocations.contains(rangePlus, &existingPlus)) { + continue; + } + const LiveRange* existing = existingPlus.liveRange(); + if (existing->hasVreg()) { + MOZ_ASSERT(existing->bundle()->allocation().toRegister() == rAlias.reg); + bool duplicate = false; + for (size_t i = 0; i < aliasedConflicting.length(); i++) { + if (aliasedConflicting[i] == existing->bundle()) { + duplicate = true; + break; + } + } + if (!duplicate && !aliasedConflicting.append(existing->bundle())) { + return false; + } + } else { + JitSpewIfEnabled(JitSpew_RegAlloc, " %s collides with fixed use %s", + rAlias.reg.name(), existing->toString().get()); + *pfixed = true; + return true; + } + } + } + + if (!aliasedConflicting.empty()) { + // One or more aliased registers is allocated to another bundle + // overlapping this one. Keep track of the conflicting set, and in the + // case of multiple conflicting sets keep track of the set with the + // lowest maximum spill weight. + + // The #ifdef guards against "unused variable 'existing'" bustage. +#ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_RegAlloc)) { + if (aliasedConflicting.length() == 1) { + LiveBundle* existing = aliasedConflicting[0]; + JitSpew(JitSpew_RegAlloc, " %s collides with %s [weight %zu]", + r.reg.name(), existing->toString().get(), + computeSpillWeight(existing)); + } else { + JitSpew(JitSpew_RegAlloc, " %s collides with the following", + r.reg.name()); + for (size_t i = 0; i < aliasedConflicting.length(); i++) { + LiveBundle* existing = aliasedConflicting[i]; + JitSpew(JitSpew_RegAlloc, " %s [weight %zu]", + existing->toString().get(), computeSpillWeight(existing)); + } + } + } +#endif + + if (conflicting.empty()) { + conflicting = std::move(aliasedConflicting); + } else { + if (maximumSpillWeight(aliasedConflicting) < + maximumSpillWeight(conflicting)) { + conflicting = std::move(aliasedConflicting); + } + } + return true; + } + + JitSpewIfEnabled(JitSpew_RegAlloc, " allocated to %s", r.reg.name()); + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + if (!alloc().ensureBallast()) { + return false; + } + LiveRangePlus rangePlus(range); + if (!r.allocations.insert(rangePlus)) { + return false; + } + } + + bundle->setAllocation(LAllocation(r.reg)); + *success = true; + return true; +} + +bool BacktrackingAllocator::tryAllocateAnyRegister( + LiveBundle* bundle, bool* success, bool* pfixed, + LiveBundleVector& conflicting) { + // Search for any available register which the bundle can be allocated to. + + LDefinition::Type type = bundle->firstRange()->vreg().type(); + + if (LDefinition::isFloatReg(type)) { + for (size_t i = AnyRegister::FirstFloatReg; i < AnyRegister::Total; i++) { + if (!LDefinition::isFloatRegCompatible(type, registers[i].reg.fpu())) { + continue; + } + if (!tryAllocateRegister(registers[i], bundle, success, pfixed, + conflicting)) { + return false; + } + if (*success) { + break; + } + } + return true; + } + + for (size_t i = 0; i < AnyRegister::FirstFloatReg; i++) { + if (!tryAllocateRegister(registers[i], bundle, success, pfixed, + conflicting)) { + return false; + } + if (*success) { + break; + } + } + return true; +} + +bool BacktrackingAllocator::evictBundle(LiveBundle* bundle) { + JitSpewIfEnabled(JitSpew_RegAlloc, + " Evicting %s [priority %zu] [weight %zu]", + bundle->toString().get(), computePriority(bundle), + computeSpillWeight(bundle)); + + AnyRegister reg(bundle->allocation().toRegister()); + PhysicalRegister& physical = registers[reg.code()]; + MOZ_ASSERT(physical.reg == reg && physical.allocatable); + + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + LiveRangePlus rangePlus(range); + physical.allocations.remove(rangePlus); + } + + bundle->setAllocation(LAllocation()); + + size_t priority = computePriority(bundle); + return allocationQueue.insert(QueueItem(bundle, priority)); +} + +bool BacktrackingAllocator::tryAllocateFixed(LiveBundle* bundle, + Requirement requirement, + bool* success, bool* pfixed, + LiveBundleVector& conflicting) { + // Spill bundles which are required to be in a certain stack slot. + if (!requirement.allocation().isRegister()) { + JitSpew(JitSpew_RegAlloc, " stack allocation requirement"); + bundle->setAllocation(requirement.allocation()); + *success = true; + return true; + } + + AnyRegister reg = requirement.allocation().toRegister(); + return tryAllocateRegister(registers[reg.code()], bundle, success, pfixed, + conflicting); +} + +bool BacktrackingAllocator::tryAllocateNonFixed(LiveBundle* bundle, + Requirement requirement, + Requirement hint, bool* success, + bool* pfixed, + LiveBundleVector& conflicting) { + // If we want, but do not require a bundle to be in a specific register, + // only look at that register for allocating and evict or spill if it is + // not available. Picking a separate register may be even worse than + // spilling, as it will still necessitate moves and will tie up more + // registers than if we spilled. + if (hint.kind() == Requirement::FIXED) { + AnyRegister reg = hint.allocation().toRegister(); + if (!tryAllocateRegister(registers[reg.code()], bundle, success, pfixed, + conflicting)) { + return false; + } + if (*success) { + return true; + } + } + + // Spill bundles which have no hint or register requirement. + if (requirement.kind() == Requirement::NONE && + hint.kind() != Requirement::REGISTER) { + JitSpew(JitSpew_RegAlloc, + " postponed spill (no hint or register requirement)"); + if (!spilledBundles.append(bundle)) { + return false; + } + *success = true; + return true; + } + + if (conflicting.empty() || minimalBundle(bundle)) { + if (!tryAllocateAnyRegister(bundle, success, pfixed, conflicting)) { + return false; + } + if (*success) { + return true; + } + } + + // Spill bundles which have no register requirement if they didn't get + // allocated. + if (requirement.kind() == Requirement::NONE) { + JitSpew(JitSpew_RegAlloc, " postponed spill (no register requirement)"); + if (!spilledBundles.append(bundle)) { + return false; + } + *success = true; + return true; + } + + // We failed to allocate this bundle. + MOZ_ASSERT(!*success); + return true; +} + +bool BacktrackingAllocator::processBundle(MIRGenerator* mir, + LiveBundle* bundle) { + JitSpewIfEnabled(JitSpew_RegAlloc, + "Allocating %s [priority %zu] [weight %zu]", + bundle->toString().get(), computePriority(bundle), + computeSpillWeight(bundle)); + + // A bundle can be processed by doing any of the following: + // + // - Assigning the bundle a register. The bundle cannot overlap any other + // bundle allocated for that physical register. + // + // - Spilling the bundle, provided it has no register uses. + // + // - Splitting the bundle into two or more bundles which cover the original + // one. The new bundles are placed back onto the priority queue for later + // processing. + // + // - Evicting one or more existing allocated bundles, and then doing one + // of the above operations. Evicted bundles are placed back on the + // priority queue. Any evicted bundles must have a lower spill weight + // than the bundle being processed. + // + // As long as this structure is followed, termination is guaranteed. + // In general, we want to minimize the amount of bundle splitting (which + // generally necessitates spills), so allocate longer lived, lower weight + // bundles first and evict and split them later if they prevent allocation + // for higher weight bundles. + + Requirement requirement, hint; + bool canAllocate = computeRequirement(bundle, &requirement, &hint); + + bool fixed; + LiveBundleVector conflicting; + for (size_t attempt = 0;; attempt++) { + if (mir->shouldCancel("Backtracking Allocation (processBundle loop)")) { + return false; + } + + if (canAllocate) { + bool success = false; + fixed = false; + conflicting.clear(); + + // Ok, let's try allocating for this bundle. + if (requirement.kind() == Requirement::FIXED) { + if (!tryAllocateFixed(bundle, requirement, &success, &fixed, + conflicting)) { + return false; + } + } else { + if (!tryAllocateNonFixed(bundle, requirement, hint, &success, &fixed, + conflicting)) { + return false; + } + } + + // If that worked, we're done! + if (success) { + return true; + } + + // If that didn't work, but we have one or more non-fixed bundles + // known to be conflicting, maybe we can evict them and try again. + if ((attempt < MAX_ATTEMPTS || minimalBundle(bundle)) && !fixed && + !conflicting.empty() && + maximumSpillWeight(conflicting) < computeSpillWeight(bundle)) { + for (size_t i = 0; i < conflicting.length(); i++) { + if (!evictBundle(conflicting[i])) { + return false; + } + } + continue; + } + } + + // A minimal bundle cannot be split any further. If we try to split it + // it at this point we will just end up with the same bundle and will + // enter an infinite loop. Weights and the initial live ranges must + // be constructed so that any minimal bundle is allocatable. + MOZ_ASSERT(!minimalBundle(bundle)); + + LiveBundle* conflict = conflicting.empty() ? nullptr : conflicting[0]; + return chooseBundleSplit(bundle, canAllocate && fixed, conflict); + } +} + +// Helper for ::tryAllocatingRegistersForSpillBundles +bool BacktrackingAllocator::spill(LiveBundle* bundle) { + JitSpew(JitSpew_RegAlloc, " Spilling bundle"); + MOZ_ASSERT(bundle->allocation().isBogus()); + + if (LiveBundle* spillParent = bundle->spillParent()) { + JitSpew(JitSpew_RegAlloc, " Using existing spill bundle"); + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + LiveRange* parentRange = spillParent->rangeFor(range->from()); + MOZ_ASSERT(parentRange->contains(range)); + MOZ_ASSERT(&range->vreg() == &parentRange->vreg()); + range->tryToMoveDefAndUsesInto(parentRange); + MOZ_ASSERT(!range->hasUses()); + range->vreg().removeRange(range); + } + return true; + } + + return bundle->spillSet()->addSpilledBundle(bundle); +} + +bool BacktrackingAllocator::tryAllocatingRegistersForSpillBundles() { + for (auto it = spilledBundles.begin(); it != spilledBundles.end(); it++) { + LiveBundle* bundle = *it; + LiveBundleVector conflicting; + bool fixed = false; + bool success = false; + + if (mir->shouldCancel("Backtracking Try Allocating Spilled Bundles")) { + return false; + } + + JitSpewIfEnabled(JitSpew_RegAlloc, "Spill or allocate %s", + bundle->toString().get()); + + if (!tryAllocateAnyRegister(bundle, &success, &fixed, conflicting)) { + return false; + } + + // If the bundle still has no register, spill the bundle. + if (!success && !spill(bundle)) { + return false; + } + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Rewriting of the LIR after bundle processing is done: // +// ::pickStackSlots // +// ::createMoveGroupsFromLiveRangeTransitions // +// ::installAllocationsInLIR // +// ::populateSafepoints // +// ::annotateMoveGroups // +// // +/////////////////////////////////////////////////////////////////////////////// + +// Helper for ::pickStackSlot +bool BacktrackingAllocator::insertAllRanges(LiveRangePlusSet& set, + LiveBundle* bundle) { + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + if (!alloc().ensureBallast()) { + return false; + } + LiveRangePlus rangePlus(range); + if (!set.insert(rangePlus)) { + return false; + } + } + return true; +} + +// Helper for ::pickStackSlots +bool BacktrackingAllocator::pickStackSlot(SpillSet* spillSet) { + // Look through all ranges that have been spilled in this set for a + // register definition which is fixed to a stack or argument slot. If we + // find one, use it for all bundles that have been spilled. tryMergeBundles + // makes sure this reuse is possible when an initial bundle contains ranges + // from multiple virtual registers. + for (size_t i = 0; i < spillSet->numSpilledBundles(); i++) { + LiveBundle* bundle = spillSet->spilledBundle(i); + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + if (range->hasDefinition()) { + LDefinition* def = range->vreg().def(); + if (def->policy() == LDefinition::FIXED) { + MOZ_ASSERT(!def->output()->isRegister()); + MOZ_ASSERT(!def->output()->isStackSlot()); + spillSet->setAllocation(*def->output()); + return true; + } + } + } + } + + LDefinition::Type type = + spillSet->spilledBundle(0)->firstRange()->vreg().type(); + + SpillSlotList* slotList; + switch (StackSlotAllocator::width(type)) { + case 4: + slotList = &normalSlots; + break; + case 8: + slotList = &doubleSlots; + break; + case 16: + slotList = &quadSlots; + break; + default: + MOZ_CRASH("Bad width"); + } + + // Maximum number of existing spill slots we will look at before giving up + // and allocating a new slot. + static const size_t MAX_SEARCH_COUNT = 10; + + size_t searches = 0; + SpillSlot* stop = nullptr; + while (!slotList->empty()) { + SpillSlot* spillSlot = *slotList->begin(); + if (!stop) { + stop = spillSlot; + } else if (stop == spillSlot) { + // We looked through every slot in the list. + break; + } + + bool success = true; + for (size_t i = 0; i < spillSet->numSpilledBundles(); i++) { + LiveBundle* bundle = spillSet->spilledBundle(i); + for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + LiveRangePlus rangePlus(range); + LiveRangePlus existingPlus; + if (spillSlot->allocated.contains(rangePlus, &existingPlus)) { + success = false; + break; + } + } + if (!success) { + break; + } + } + if (success) { + // We can reuse this physical stack slot for the new bundles. + // Update the allocated ranges for the slot. + for (size_t i = 0; i < spillSet->numSpilledBundles(); i++) { + LiveBundle* bundle = spillSet->spilledBundle(i); + if (!insertAllRanges(spillSlot->allocated, bundle)) { + return false; + } + } + spillSet->setAllocation(spillSlot->alloc); + return true; + } + + // On a miss, move the spill to the end of the list. This will cause us + // to make fewer attempts to allocate from slots with a large and + // highly contended range. + slotList->popFront(); + slotList->pushBack(spillSlot); + + if (++searches == MAX_SEARCH_COUNT) { + break; + } + } + + // We need a new physical stack slot. + uint32_t stackSlot = stackSlotAllocator.allocateSlot(type); + + SpillSlot* spillSlot = + new (alloc().fallible()) SpillSlot(stackSlot, alloc().lifoAlloc()); + if (!spillSlot) { + return false; + } + + for (size_t i = 0; i < spillSet->numSpilledBundles(); i++) { + LiveBundle* bundle = spillSet->spilledBundle(i); + if (!insertAllRanges(spillSlot->allocated, bundle)) { + return false; + } + } + + spillSet->setAllocation(spillSlot->alloc); + + slotList->pushFront(spillSlot); + return true; +} + +bool BacktrackingAllocator::pickStackSlots() { + for (size_t i = 1; i < graph.numVirtualRegisters(); i++) { + VirtualRegister& reg = vregs[i]; + + if (mir->shouldCancel("Backtracking Pick Stack Slots")) { + return false; + } + + for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + LiveBundle* bundle = range->bundle(); + + if (bundle->allocation().isBogus()) { + if (!pickStackSlot(bundle->spillSet())) { + return false; + } + MOZ_ASSERT(!bundle->allocation().isBogus()); + } + } + } + + return true; +} + +// Helper for ::createMoveGroupsFromLiveRangeTransitions +bool BacktrackingAllocator::moveAtEdge(LBlock* predecessor, LBlock* successor, + LiveRange* from, LiveRange* to, + LDefinition::Type type) { + if (successor->mir()->numPredecessors() > 1) { + MOZ_ASSERT(predecessor->mir()->numSuccessors() == 1); + return moveAtExit(predecessor, from, to, type); + } + + return moveAtEntry(successor, from, to, type); +} + +// Helper for ::createMoveGroupsFromLiveRangeTransitions +bool BacktrackingAllocator::deadRange(LiveRange* range) { + // Check for direct uses of this range. + if (range->hasUses() || range->hasDefinition()) { + return false; + } + + CodePosition start = range->from(); + LNode* ins = insData[start]; + if (start == entryOf(ins->block())) { + return false; + } + + VirtualRegister& reg = range->vreg(); + + // Check if there are later ranges for this vreg. + LiveRange::RegisterLinkIterator iter = reg.rangesBegin(range); + for (iter++; iter; iter++) { + LiveRange* laterRange = LiveRange::get(*iter); + if (laterRange->from() > range->from()) { + return false; + } + } + + // Check if this range ends at a loop backedge. + LNode* last = insData[range->to().previous()]; + if (last->isGoto() && + last->toGoto()->target()->id() < last->block()->mir()->id()) { + return false; + } + + // Check if there are phis which this vreg flows to. + if (reg.usedByPhi()) { + return false; + } + + return true; +} + +bool BacktrackingAllocator::createMoveGroupsFromLiveRangeTransitions() { + // Add moves to handle changing assignments for vregs over their lifetime. + JitSpew(JitSpew_RegAlloc, "ResolveControlFlow: begin"); + + JitSpew(JitSpew_RegAlloc, + " ResolveControlFlow: adding MoveGroups within blocks"); + + // Look for places where a register's assignment changes in the middle of a + // basic block. + MOZ_ASSERT(!vregs[0u].hasRanges()); + for (size_t i = 1; i < graph.numVirtualRegisters(); i++) { + VirtualRegister& reg = vregs[i]; + + if (mir->shouldCancel( + "Backtracking Resolve Control Flow (vreg outer loop)")) { + return false; + } + + for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter;) { + LiveRange* range = LiveRange::get(*iter); + + if (mir->shouldCancel( + "Backtracking Resolve Control Flow (vreg inner loop)")) { + return false; + } + + // Remove ranges which will never be used. + if (deadRange(range)) { + reg.removeRangeAndIncrement(iter); + continue; + } + + // The range which defines the register does not have a predecessor + // to add moves from. + if (range->hasDefinition()) { + iter++; + continue; + } + + // Ignore ranges that start at block boundaries. We will handle + // these in the next phase. + CodePosition start = range->from(); + LNode* ins = insData[start]; + if (start == entryOf(ins->block())) { + iter++; + continue; + } + + // If we already saw a range which covers the start of this range + // and has the same allocation, we don't need an explicit move at + // the start of this range. + bool skip = false; + for (LiveRange::RegisterLinkIterator prevIter = reg.rangesBegin(); + prevIter != iter; prevIter++) { + LiveRange* prevRange = LiveRange::get(*prevIter); + if (prevRange->covers(start) && prevRange->bundle()->allocation() == + range->bundle()->allocation()) { + skip = true; + break; + } + } + if (skip) { + iter++; + continue; + } + + if (!alloc().ensureBallast()) { + return false; + } + + LiveRange* predecessorRange = + reg.rangeFor(start.previous(), /* preferRegister = */ true); + if (start.subpos() == CodePosition::INPUT) { + JitSpewIfEnabled(JitSpew_RegAlloc, " moveInput (%s) <- (%s)", + range->toString().get(), + predecessorRange->toString().get()); + if (!moveInput(ins->toInstruction(), predecessorRange, range, + reg.type())) { + return false; + } + } else { + JitSpew(JitSpew_RegAlloc, " (moveAfter)"); + if (!moveAfter(ins->toInstruction(), predecessorRange, range, + reg.type())) { + return false; + } + } + + iter++; + } + } + + JitSpew(JitSpew_RegAlloc, + " ResolveControlFlow: adding MoveGroups for phi nodes"); + + for (size_t i = 0; i < graph.numBlocks(); i++) { + if (mir->shouldCancel("Backtracking Resolve Control Flow (block loop)")) { + return false; + } + + LBlock* successor = graph.getBlock(i); + MBasicBlock* mSuccessor = successor->mir(); + if (mSuccessor->numPredecessors() < 1) { + continue; + } + + // Resolve phis to moves. + for (size_t j = 0; j < successor->numPhis(); j++) { + LPhi* phi = successor->getPhi(j); + MOZ_ASSERT(phi->numDefs() == 1); + LDefinition* def = phi->getDef(0); + VirtualRegister& reg = vreg(def); + LiveRange* to = reg.rangeFor(entryOf(successor)); + MOZ_ASSERT(to); + + for (size_t k = 0; k < mSuccessor->numPredecessors(); k++) { + LBlock* predecessor = mSuccessor->getPredecessor(k)->lir(); + MOZ_ASSERT(predecessor->mir()->numSuccessors() == 1); + + LAllocation* input = phi->getOperand(k); + LiveRange* from = vreg(input).rangeFor(exitOf(predecessor), + /* preferRegister = */ true); + MOZ_ASSERT(from); + + if (!alloc().ensureBallast()) { + return false; + } + + // Note: we have to use moveAtEdge both here and below (for edge + // resolution) to avoid conflicting moves. See bug 1493900. + JitSpew(JitSpew_RegAlloc, " (moveAtEdge#1)"); + if (!moveAtEdge(predecessor, successor, from, to, def->type())) { + return false; + } + } + } + } + + JitSpew(JitSpew_RegAlloc, + " ResolveControlFlow: adding MoveGroups to fix conflicted edges"); + + // Add moves to resolve graph edges with different allocations at their + // source and target. + for (size_t i = 1; i < graph.numVirtualRegisters(); i++) { + VirtualRegister& reg = vregs[i]; + for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter; + iter++) { + LiveRange* targetRange = LiveRange::get(*iter); + + size_t firstBlockId = insData[targetRange->from()]->block()->mir()->id(); + if (!targetRange->covers(entryOf(graph.getBlock(firstBlockId)))) { + firstBlockId++; + } + for (size_t id = firstBlockId; id < graph.numBlocks(); id++) { + LBlock* successor = graph.getBlock(id); + if (!targetRange->covers(entryOf(successor))) { + break; + } + + BitSet& live = liveIn[id]; + if (!live.contains(i)) { + continue; + } + + for (size_t j = 0; j < successor->mir()->numPredecessors(); j++) { + LBlock* predecessor = successor->mir()->getPredecessor(j)->lir(); + if (targetRange->covers(exitOf(predecessor))) { + continue; + } + + if (!alloc().ensureBallast()) { + return false; + } + JitSpew(JitSpew_RegAlloc, " (moveAtEdge#2)"); + LiveRange* from = reg.rangeFor(exitOf(predecessor), true); + if (!moveAtEdge(predecessor, successor, from, targetRange, + reg.type())) { + return false; + } + } + } + } + } + + JitSpew(JitSpew_RegAlloc, "ResolveControlFlow: end"); + return true; +} + +// Helper for ::addLiveRegistersForRange +size_t BacktrackingAllocator::findFirstNonCallSafepoint(CodePosition from) { + size_t i = 0; + for (; i < graph.numNonCallSafepoints(); i++) { + const LInstruction* ins = graph.getNonCallSafepoint(i); + if (from <= inputOf(ins)) { + break; + } + } + return i; +} + +// Helper for ::installAllocationsInLIR +void BacktrackingAllocator::addLiveRegistersForRange(VirtualRegister& reg, + LiveRange* range) { + // Fill in the live register sets for all non-call safepoints. + LAllocation a = range->bundle()->allocation(); + if (!a.isRegister()) { + return; + } + + // Don't add output registers to the safepoint. + CodePosition start = range->from(); + if (range->hasDefinition() && !reg.isTemp()) { +#ifdef CHECK_OSIPOINT_REGISTERS + // We don't add the output register to the safepoint, + // but it still might get added as one of the inputs. + // So eagerly add this reg to the safepoint clobbered registers. + if (reg.ins()->isInstruction()) { + if (LSafepoint* safepoint = reg.ins()->toInstruction()->safepoint()) { + safepoint->addClobberedRegister(a.toRegister()); + } + } +#endif + start = start.next(); + } + + size_t i = findFirstNonCallSafepoint(start); + for (; i < graph.numNonCallSafepoints(); i++) { + LInstruction* ins = graph.getNonCallSafepoint(i); + CodePosition pos = inputOf(ins); + + // Safepoints are sorted, so we can shortcut out of this loop + // if we go out of range. + if (range->to() <= pos) { + break; + } + + MOZ_ASSERT(range->covers(pos)); + + LSafepoint* safepoint = ins->safepoint(); + safepoint->addLiveRegister(a.toRegister()); + +#ifdef CHECK_OSIPOINT_REGISTERS + if (reg.isTemp()) { + safepoint->addClobberedRegister(a.toRegister()); + } +#endif + } +} + +// Helper for ::installAllocationsInLIR +static inline size_t NumReusingDefs(LInstruction* ins) { + size_t num = 0; + for (size_t i = 0; i < ins->numDefs(); i++) { + LDefinition* def = ins->getDef(i); + if (def->policy() == LDefinition::MUST_REUSE_INPUT) { + num++; + } + } + return num; +} + +bool BacktrackingAllocator::installAllocationsInLIR() { + JitSpew(JitSpew_RegAlloc, "Installing Allocations"); + + MOZ_ASSERT(!vregs[0u].hasRanges()); + for (size_t i = 1; i < graph.numVirtualRegisters(); i++) { + VirtualRegister& reg = vregs[i]; + + if (mir->shouldCancel("Backtracking Install Allocations (main loop)")) { + return false; + } + + for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + + if (range->hasDefinition()) { + reg.def()->setOutput(range->bundle()->allocation()); + if (reg.ins()->recoversInput()) { + LSnapshot* snapshot = reg.ins()->toInstruction()->snapshot(); + for (size_t i = 0; i < snapshot->numEntries(); i++) { + LAllocation* entry = snapshot->getEntry(i); + if (entry->isUse() && + entry->toUse()->policy() == LUse::RECOVERED_INPUT) { + *entry = *reg.def()->output(); + } + } + } + } + + for (UsePositionIterator iter(range->usesBegin()); iter; iter++) { + LAllocation* alloc = iter->use(); + *alloc = range->bundle()->allocation(); + + // For any uses which feed into MUST_REUSE_INPUT definitions, + // add copies if the use and def have different allocations. + LNode* ins = insData[iter->pos]; + if (LDefinition* def = FindReusingDefOrTemp(ins, alloc)) { + LiveRange* outputRange = vreg(def).rangeFor(outputOf(ins)); + LAllocation res = outputRange->bundle()->allocation(); + LAllocation sourceAlloc = range->bundle()->allocation(); + + if (res != *alloc) { + if (!this->alloc().ensureBallast()) { + return false; + } + if (NumReusingDefs(ins->toInstruction()) <= 1) { + LMoveGroup* group = getInputMoveGroup(ins->toInstruction()); + if (!group->addAfter(sourceAlloc, res, reg.type())) { + return false; + } + } else { + LMoveGroup* group = getFixReuseMoveGroup(ins->toInstruction()); + if (!group->add(sourceAlloc, res, reg.type())) { + return false; + } + } + *alloc = res; + } + } + } + + addLiveRegistersForRange(reg, range); + } + } + + graph.setLocalSlotsSize(stackSlotAllocator.stackHeight()); + return true; +} + +// Helper for ::populateSafepoints +size_t BacktrackingAllocator::findFirstSafepoint(CodePosition pos, + size_t startFrom) { + size_t i = startFrom; + for (; i < graph.numSafepoints(); i++) { + LInstruction* ins = graph.getSafepoint(i); + if (pos <= inputOf(ins)) { + break; + } + } + return i; +} + +// Helper for ::populateSafepoints +static inline bool IsNunbox(VirtualRegister& reg) { +#ifdef JS_NUNBOX32 + return reg.type() == LDefinition::TYPE || reg.type() == LDefinition::PAYLOAD; +#else + return false; +#endif +} + +// Helper for ::populateSafepoints +static inline bool IsSlotsOrElements(VirtualRegister& reg) { + return reg.type() == LDefinition::SLOTS; +} + +// Helper for ::populateSafepoints +static inline bool IsTraceable(VirtualRegister& reg) { + if (reg.type() == LDefinition::OBJECT) { + return true; + } +#ifdef JS_PUNBOX64 + if (reg.type() == LDefinition::BOX) { + return true; + } +#endif + if (reg.type() == LDefinition::STACKRESULTS) { + MOZ_ASSERT(reg.def()); + const LStackArea* alloc = reg.def()->output()->toStackArea(); + for (auto iter = alloc->results(); iter; iter.next()) { + if (iter.isGcPointer()) { + return true; + } + } + } + return false; +} + +bool BacktrackingAllocator::populateSafepoints() { + JitSpew(JitSpew_RegAlloc, "Populating Safepoints"); + + size_t firstSafepoint = 0; + + MOZ_ASSERT(!vregs[0u].def()); + for (uint32_t i = 1; i < graph.numVirtualRegisters(); i++) { + VirtualRegister& reg = vregs[i]; + + if (!reg.def() || + (!IsTraceable(reg) && !IsSlotsOrElements(reg) && !IsNunbox(reg))) { + continue; + } + + firstSafepoint = findFirstSafepoint(inputOf(reg.ins()), firstSafepoint); + if (firstSafepoint >= graph.numSafepoints()) { + break; + } + + for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter; + iter++) { + LiveRange* range = LiveRange::get(*iter); + + for (size_t j = firstSafepoint; j < graph.numSafepoints(); j++) { + LInstruction* ins = graph.getSafepoint(j); + + if (!range->covers(inputOf(ins))) { + if (inputOf(ins) >= range->to()) { + break; + } + continue; + } + + // Include temps but not instruction outputs. Also make sure + // MUST_REUSE_INPUT is not used with gcthings or nunboxes, or + // we would have to add the input reg to this safepoint. + if (ins == reg.ins() && !reg.isTemp()) { + DebugOnly def = reg.def(); + MOZ_ASSERT_IF(def->policy() == LDefinition::MUST_REUSE_INPUT, + def->type() == LDefinition::GENERAL || + def->type() == LDefinition::INT32 || + def->type() == LDefinition::FLOAT32 || + def->type() == LDefinition::DOUBLE || + def->type() == LDefinition::SIMD128); + continue; + } + + LSafepoint* safepoint = ins->safepoint(); + + LAllocation a = range->bundle()->allocation(); + if (a.isGeneralReg() && ins->isCall()) { + continue; + } + + switch (reg.type()) { + case LDefinition::OBJECT: + if (!safepoint->addGcPointer(a)) { + return false; + } + break; + case LDefinition::SLOTS: + if (!safepoint->addSlotsOrElementsPointer(a)) { + return false; + } + break; + case LDefinition::STACKRESULTS: { + MOZ_ASSERT(a.isStackArea()); + for (auto iter = a.toStackArea()->results(); iter; iter.next()) { + if (iter.isGcPointer()) { + if (!safepoint->addGcPointer(iter.alloc())) { + return false; + } + } + } + break; + } +#ifdef JS_NUNBOX32 + case LDefinition::TYPE: + if (!safepoint->addNunboxType(i, a)) { + return false; + } + break; + case LDefinition::PAYLOAD: + if (!safepoint->addNunboxPayload(i, a)) { + return false; + } + break; +#else + case LDefinition::BOX: + if (!safepoint->addBoxedValue(a)) { + return false; + } + break; +#endif + default: + MOZ_CRASH("Bad register type"); + } + } + } + } + + return true; +} + +bool BacktrackingAllocator::annotateMoveGroups() { + // Annotate move groups in the LIR graph with any register that is not + // allocated at that point and can be used as a scratch register. This is + // only required for x86, as other platforms always have scratch registers + // available for use. +#ifdef JS_CODEGEN_X86 + LiveRange* range = LiveRange::FallibleNew(alloc(), nullptr, CodePosition(), + CodePosition().next()); + if (!range) { + return false; + } + + for (size_t i = 0; i < graph.numBlocks(); i++) { + if (mir->shouldCancel("Backtracking Annotate Move Groups")) { + return false; + } + + LBlock* block = graph.getBlock(i); + LInstruction* last = nullptr; + for (LInstructionIterator iter = block->begin(); iter != block->end(); + ++iter) { + if (iter->isMoveGroup()) { + CodePosition from = last ? outputOf(last) : entryOf(block); + range->setTo(from.next()); + range->setFrom(from); + + for (size_t i = 0; i < AnyRegister::Total; i++) { + PhysicalRegister& reg = registers[i]; + if (reg.reg.isFloat() || !reg.allocatable) { + continue; + } + + // This register is unavailable for use if (a) it is in use + // by some live range immediately before the move group, + // or (b) it is an operand in one of the group's moves. The + // latter case handles live ranges which end immediately + // before the move group or start immediately after. + // For (b) we need to consider move groups immediately + // preceding or following this one. + + if (iter->toMoveGroup()->uses(reg.reg.gpr())) { + continue; + } + bool found = false; + LInstructionIterator niter(iter); + for (niter++; niter != block->end(); niter++) { + if (niter->isMoveGroup()) { + if (niter->toMoveGroup()->uses(reg.reg.gpr())) { + found = true; + break; + } + } else { + break; + } + } + if (iter != block->begin()) { + LInstructionIterator riter(iter); + do { + riter--; + if (riter->isMoveGroup()) { + if (riter->toMoveGroup()->uses(reg.reg.gpr())) { + found = true; + break; + } + } else { + break; + } + } while (riter != block->begin()); + } + + if (found) { + continue; + } + LiveRangePlus existingPlus; + LiveRangePlus rangePlus(range); + if (reg.allocations.contains(rangePlus, &existingPlus)) { + continue; + } + + iter->toMoveGroup()->setScratchRegister(reg.reg.gpr()); + break; + } + } else { + last = *iter; + } + } + } +#endif + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// // +// Debug-printing support // +// // +/////////////////////////////////////////////////////////////////////////////// + +#ifdef JS_JITSPEW + +UniqueChars LiveRange::toString() const { + AutoEnterOOMUnsafeRegion oomUnsafe; + + UniqueChars buf = JS_smprintf("v%u %u-%u", hasVreg() ? vreg().vreg() : 0, + from().bits(), to().bits() - 1); + + if (buf && bundle() && !bundle()->allocation().isBogus()) { + buf = JS_sprintf_append(std::move(buf), " %s", + bundle()->allocation().toString().get()); + } + + buf = JS_sprintf_append(std::move(buf), " {"); + + if (buf && hasDefinition()) { + buf = JS_sprintf_append(std::move(buf), " %u_def", from().bits()); + if (hasVreg()) { + // If the definition has a fixed requirement, print it too. + const LDefinition* def = vreg().def(); + LDefinition::Policy policy = def->policy(); + if (policy == LDefinition::FIXED || policy == LDefinition::STACK) { + if (buf) { + buf = JS_sprintf_append(std::move(buf), ":F:%s", + def->output()->toString().get()); + } + } + } + } + + for (UsePositionIterator iter = usesBegin(); buf && iter; iter++) { + buf = JS_sprintf_append(std::move(buf), " %u_%s", iter->pos.bits(), + iter->use()->toString().get()); + } + + buf = JS_sprintf_append(std::move(buf), " }"); + + if (!buf) { + oomUnsafe.crash("LiveRange::toString()"); + } + + return buf; +} + +UniqueChars LiveBundle::toString() const { + AutoEnterOOMUnsafeRegion oomUnsafe; + + UniqueChars buf = JS_smprintf("LB%u(", debugId()); + + if (buf) { + if (spillParent()) { + buf = JS_sprintf_append(std::move(buf), "parent=LB%u", + spillParent()->debugId()); + } else { + buf = JS_sprintf_append(std::move(buf), "parent=none"); + } + } + + for (LiveRange::BundleLinkIterator iter = rangesBegin(); buf && iter; + iter++) { + if (buf) { + buf = JS_sprintf_append(std::move(buf), "%s %s", + (iter == rangesBegin()) ? "" : " ##", + LiveRange::get(*iter)->toString().get()); + } + } + + if (buf) { + buf = JS_sprintf_append(std::move(buf), ")"); + } + + if (!buf) { + oomUnsafe.crash("LiveBundle::toString()"); + } + + return buf; +} + +void BacktrackingAllocator::dumpLiveRangesByVReg(const char* who) { + MOZ_ASSERT(!vregs[0u].hasRanges()); + + JitSpewCont(JitSpew_RegAlloc, "\n"); + JitSpew(JitSpew_RegAlloc, "Live ranges by virtual register (%s):", who); + + for (uint32_t i = 1; i < graph.numVirtualRegisters(); i++) { + JitSpewHeader(JitSpew_RegAlloc); + JitSpewCont(JitSpew_RegAlloc, " "); + VirtualRegister& reg = vregs[i]; + for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter; + iter++) { + if (iter != reg.rangesBegin()) { + JitSpewCont(JitSpew_RegAlloc, " ## "); + } + JitSpewCont(JitSpew_RegAlloc, "%s", + LiveRange::get(*iter)->toString().get()); + } + JitSpewCont(JitSpew_RegAlloc, "\n"); + } +} + +void BacktrackingAllocator::dumpLiveRangesByBundle(const char* who) { + MOZ_ASSERT(!vregs[0u].hasRanges()); + + JitSpewCont(JitSpew_RegAlloc, "\n"); + JitSpew(JitSpew_RegAlloc, "Live ranges by bundle (%s):", who); + + for (uint32_t i = 1; i < graph.numVirtualRegisters(); i++) { + VirtualRegister& reg = vregs[i]; + for (LiveRange::RegisterLinkIterator baseIter = reg.rangesBegin(); baseIter; + baseIter++) { + LiveRange* range = LiveRange::get(*baseIter); + LiveBundle* bundle = range->bundle(); + if (range == bundle->firstRange()) { + JitSpew(JitSpew_RegAlloc, " %s", bundle->toString().get()); + } + } + } +} + +void BacktrackingAllocator::dumpAllocations() { + JitSpew(JitSpew_RegAlloc, "Allocations:"); + + dumpLiveRangesByBundle("in dumpAllocations()"); + + JitSpewCont(JitSpew_RegAlloc, "\n"); + JitSpew(JitSpew_RegAlloc, "Allocations by physical register:"); + + for (size_t i = 0; i < AnyRegister::Total; i++) { + if (registers[i].allocatable && !registers[i].allocations.empty()) { + JitSpewHeader(JitSpew_RegAlloc); + JitSpewCont(JitSpew_RegAlloc, " %s:", AnyRegister::FromCode(i).name()); + bool first = true; + LiveRangePlusSet::Iter lrpIter(®isters[i].allocations); + while (lrpIter.hasMore()) { + LiveRange* range = lrpIter.next().liveRange(); + if (first) { + first = false; + } else { + fprintf(stderr, " /"); + } + fprintf(stderr, " %s", range->toString().get()); + } + JitSpewCont(JitSpew_RegAlloc, "\n"); + } + } + + JitSpewCont(JitSpew_RegAlloc, "\n"); +} + +#endif // JS_JITSPEW + +/////////////////////////////////////////////////////////////////////////////// +// // +// Top level of the register allocation machinery // +// // +/////////////////////////////////////////////////////////////////////////////// + +bool BacktrackingAllocator::go() { + JitSpewCont(JitSpew_RegAlloc, "\n"); + JitSpew(JitSpew_RegAlloc, "Beginning register allocation"); + + JitSpewCont(JitSpew_RegAlloc, "\n"); + if (JitSpewEnabled(JitSpew_RegAlloc)) { + dumpInstructions("(Pre-allocation LIR)"); + } + + if (!init()) { + return false; + } + + if (!buildLivenessInfo()) { + return false; + } + +#ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_RegAlloc)) { + dumpLiveRangesByVReg("after liveness analysis"); + } +#endif + + if (!allocationQueue.reserve(graph.numVirtualRegisters() * 3 / 2)) { + return false; + } + + JitSpewCont(JitSpew_RegAlloc, "\n"); + JitSpew(JitSpew_RegAlloc, "Beginning grouping and queueing registers"); + if (!mergeAndQueueRegisters()) { + return false; + } + JitSpew(JitSpew_RegAlloc, "Completed grouping and queueing registers"); + +#ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_RegAlloc)) { + dumpLiveRangesByBundle("after grouping/queueing regs"); + } +#endif + + // There now follow two allocation loops, which are really the heart of the + // allocator. First, the "main" allocation loop. This does almost all of + // the allocation work, by repeatedly pulling bundles out of + // ::allocationQueue and calling ::processBundle on it, until there are no + // bundles left in the queue. Note that ::processBundle can add new smaller + // bundles to the queue if it needs to split or spill a bundle. + // + // For each bundle in turn pulled out of ::allocationQueue, ::processBundle: + // + // * calls ::computeRequirement to discover the overall constraint for the + // bundle. + // + // * tries to find a register for it, by calling either ::tryAllocateFixed or + // ::tryAllocateNonFixed. + // + // * if that fails, but ::tryAllocateFixed / ::tryAllocateNonFixed indicate + // that there is some other bundle with lower spill weight that can be + // evicted, then that bundle is evicted (hence, put back into + // ::allocationQueue), and we try again. + // + // * at most MAX_ATTEMPTS may be made. + // + // * If that still fails to find a register, then the bundle is handed off to + // ::chooseBundleSplit. That will choose to either split the bundle, + // yielding multiple pieces which are put back into ::allocationQueue, or + // it will spill the bundle. Note that the same mechanism applies to both; + // there's no clear boundary between splitting and spilling, because + // spilling can be interpreted as an extreme form of splitting. + // + // ::processBundle and its callees contains much gnarly and logic which isn't + // easy to understand, particularly in the area of how eviction candidates + // are chosen. But it works well enough, and tinkering doesn't seem to + // improve the resulting allocations. More important is the splitting logic, + // because that controls where spill/reload instructions are placed. + // + // Eventually ::allocationQueue becomes empty, and each LiveBundle has either + // been allocated a register or is marked for spilling. In the latter case + // it will have been added to ::spilledBundles. + + JitSpewCont(JitSpew_RegAlloc, "\n"); + JitSpew(JitSpew_RegAlloc, "Beginning main allocation loop"); + JitSpewCont(JitSpew_RegAlloc, "\n"); + + // Allocate, spill and split bundles until finished. + while (!allocationQueue.empty()) { + if (mir->shouldCancel("Backtracking Allocation")) { + return false; + } + + QueueItem item = allocationQueue.removeHighest(); + if (!processBundle(mir, item.bundle)) { + return false; + } + } + + // And here's the second allocation loop (hidden inside + // ::tryAllocatingRegistersForSpillBundles). It makes one last attempt to + // find a register for each spill bundle. There's no attempt to free up + // registers by eviction. In at least 99% of cases this attempt fails, in + // which case the bundle is handed off to ::spill. The lucky remaining 1% + // get a register. Unfortunately this scheme interacts badly with the + // splitting strategy, leading to excessive register-to-register copying in + // some very simple cases. See bug 1752520. + // + // A modest but probably worthwhile amount of allocation time can be saved by + // making ::tryAllocatingRegistersForSpillBundles use specialised versions of + // ::tryAllocateAnyRegister and its callees, that don't bother to create sets + // of conflicting bundles. Creating those sets is expensive and, here, + // pointless, since we're not going to do any eviction based on them. This + // refinement is implemented in the un-landed patch at bug 1758274 comment + // 15. + + JitSpewCont(JitSpew_RegAlloc, "\n"); + JitSpew(JitSpew_RegAlloc, + "Main allocation loop complete; " + "beginning spill-bundle allocation loop"); + JitSpewCont(JitSpew_RegAlloc, "\n"); + + if (!tryAllocatingRegistersForSpillBundles()) { + return false; + } + + JitSpewCont(JitSpew_RegAlloc, "\n"); + JitSpew(JitSpew_RegAlloc, "Spill-bundle allocation loop complete"); + JitSpewCont(JitSpew_RegAlloc, "\n"); + + if (!pickStackSlots()) { + return false; + } + +#ifdef JS_JITSPEW + if (JitSpewEnabled(JitSpew_RegAlloc)) { + dumpAllocations(); + } +#endif + + if (!createMoveGroupsFromLiveRangeTransitions()) { + return false; + } + + if (!installAllocationsInLIR()) { + return false; + } + + if (!populateSafepoints()) { + return false; + } + + if (!annotateMoveGroups()) { + return false; + } + + JitSpewCont(JitSpew_RegAlloc, "\n"); + if (JitSpewEnabled(JitSpew_RegAlloc)) { + dumpInstructions("(Post-allocation LIR)"); + } + + JitSpew(JitSpew_RegAlloc, "Finished register allocation"); + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// // +/////////////////////////////////////////////////////////////////////////////// diff --git a/js/src/jit/BacktrackingAllocator.h b/js/src/jit/BacktrackingAllocator.h new file mode 100644 index 0000000000..366aa0d16c --- /dev/null +++ b/js/src/jit/BacktrackingAllocator.h @@ -0,0 +1,844 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_BacktrackingAllocator_h +#define jit_BacktrackingAllocator_h + +#include "mozilla/Array.h" +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" + +#include "ds/AvlTree.h" +#include "ds/PriorityQueue.h" +#include "jit/RegisterAllocator.h" +#include "jit/StackSlotAllocator.h" + +// Gives better traces in Nightly/debug builds (could be EARLY_BETA_OR_EARLIER) +#if defined(NIGHTLY_BUILD) || defined(DEBUG) +# define AVOID_INLINE_FOR_DEBUGGING MOZ_NEVER_INLINE +#else +# define AVOID_INLINE_FOR_DEBUGGING +#endif + +// Backtracking priority queue based register allocator based on that described +// in the following blog post: +// +// http://blog.llvm.org/2011/09/greedy-register-allocation-in-llvm-30.html + +namespace js { +namespace jit { + +class Requirement { + public: + enum Kind { NONE, REGISTER, FIXED }; + + Requirement() : kind_(NONE) {} + + explicit Requirement(Kind kind) : kind_(kind) { + // FIXED has a dedicated constructor. + MOZ_ASSERT(kind != FIXED); + } + + explicit Requirement(LAllocation fixed) : kind_(FIXED), allocation_(fixed) { + MOZ_ASSERT(!fixed.isBogus() && !fixed.isUse()); + } + + Kind kind() const { return kind_; } + + LAllocation allocation() const { + MOZ_ASSERT(!allocation_.isBogus() && !allocation_.isUse()); + return allocation_; + } + + [[nodiscard]] bool merge(const Requirement& newRequirement) { + // Merge newRequirement with any existing requirement, returning false + // if the new and old requirements conflict. + + if (newRequirement.kind() == Requirement::FIXED) { + if (kind() == Requirement::FIXED) { + return newRequirement.allocation() == allocation(); + } + *this = newRequirement; + return true; + } + + MOZ_ASSERT(newRequirement.kind() == Requirement::REGISTER); + if (kind() == Requirement::FIXED) { + return allocation().isRegister(); + } + + *this = newRequirement; + return true; + } + + private: + Kind kind_; + LAllocation allocation_; +}; + +struct UsePosition : public TempObject, + public InlineForwardListNode { + private: + // A UsePosition is an LUse* with a CodePosition. UsePosition also has an + // optimization that allows access to the associated LUse::Policy without + // dereferencing memory: the policy is encoded in the low bits of the LUse*. + // + // Note however that because LUse* is uintptr_t-aligned, on 32-bit systems + // there are only 4 encodable values, for more than 4 use policies; in that + // case we allocate the common LUse::ANY, LUse::REGISTER, and LUse::FIXED use + // policies to tags, and use tag 0x3 to indicate that dereferencing the LUse + // is necessary to get the policy (KEEPALIVE or STACK, in that case). + uintptr_t use_; + static_assert(LUse::ANY < 0x3, + "LUse::ANY can be represented in low tag on 32-bit systems"); + static_assert(LUse::REGISTER < 0x3, + "LUse::REGISTER can be represented in tag on 32-bit systems"); + static_assert(LUse::FIXED < 0x3, + "LUse::FIXED can be represented in tag on 32-bit systems"); + + static constexpr uintptr_t PolicyMask = sizeof(uintptr_t) - 1; + static constexpr uintptr_t UseMask = ~PolicyMask; + + void setUse(LUse* use) { + // RECOVERED_INPUT is used by snapshots and ignored when building the + // liveness information. Thus we can safely assume that no such value + // would be seen. + MOZ_ASSERT(use->policy() != LUse::RECOVERED_INPUT); + + uintptr_t policyBits = use->policy(); +#ifndef JS_64BIT + // On a 32-bit machine, LUse::KEEPALIVE and LUse::STACK are accessed by + // dereferencing the use pointer. + if (policyBits >= PolicyMask) { + policyBits = PolicyMask; + } +#endif + use_ = uintptr_t(use) | policyBits; + MOZ_ASSERT(use->policy() == usePolicy()); + } + + public: + CodePosition pos; + + LUse* use() const { return reinterpret_cast(use_ & UseMask); } + + LUse::Policy usePolicy() const { + uintptr_t bits = use_ & PolicyMask; +#ifndef JS_64BIT + // On 32-bit machines, reach out to memory if it's LUse::KEEPALIVE or + // LUse::STACK. + if (bits == PolicyMask) { + return use()->policy(); + } +#endif + LUse::Policy policy = LUse::Policy(bits); + MOZ_ASSERT(use()->policy() == policy); + return policy; + } + + UsePosition(LUse* use, CodePosition pos) : pos(pos) { + // Verify that the usedAtStart() flag is consistent with the + // subposition. For now ignore fixed registers, because they + // are handled specially around calls. + MOZ_ASSERT_IF(!use->isFixedRegister(), + pos.subpos() == (use->usedAtStart() ? CodePosition::INPUT + : CodePosition::OUTPUT)); + setUse(use); + } +}; + +using UsePositionIterator = InlineForwardListIterator; + +// Backtracking allocator data structures overview. +// +// LiveRange: A continuous range of positions where a virtual register is live. +// LiveBundle: A set of LiveRanges which do not overlap. +// VirtualRegister: A set of all LiveRanges used for some LDefinition. +// +// The allocator first performs a liveness ananlysis on the LIR graph which +// constructs LiveRanges for each VirtualRegister, determining where the +// registers are live. +// +// The ranges are then bundled together according to heuristics, and placed on +// the allocation queue. +// +// As bundles are removed from the allocation queue, we attempt to find a +// physical register or stack slot allocation for all ranges in the removed +// bundle, possibly evicting already-allocated bundles. See processBundle() +// for details. +// +// If we are not able to allocate a bundle, it is split according to heuristics +// into two or more smaller bundles which cover all the ranges of the original. +// These smaller bundles are then allocated independently. + +class LiveBundle; +class VirtualRegister; + +class LiveRange : public TempObject { + public: + // Linked lists are used to keep track of the ranges in each LiveBundle and + // VirtualRegister. Since a LiveRange may be in two lists simultaneously, use + // these auxiliary classes to keep things straight. + class BundleLink : public InlineForwardListNode {}; + class RegisterLink : public InlineForwardListNode {}; + + using BundleLinkIterator = InlineForwardListIterator; + using RegisterLinkIterator = InlineForwardListIterator; + + // Links in the lists in LiveBundle and VirtualRegister. + BundleLink bundleLink; + RegisterLink registerLink; + + static LiveRange* get(BundleLink* link) { + return reinterpret_cast(reinterpret_cast(link) - + offsetof(LiveRange, bundleLink)); + } + static LiveRange* get(RegisterLink* link) { + return reinterpret_cast(reinterpret_cast(link) - + offsetof(LiveRange, registerLink)); + } + + struct Range { + // The beginning of this range, inclusive. + CodePosition from; + + // The end of this range, exclusive. + CodePosition to; + + Range() = default; + + Range(CodePosition from, CodePosition to) : from(from), to(to) { + MOZ_ASSERT(!empty()); + } + + bool empty() { + MOZ_ASSERT(from <= to); + return from == to; + } + }; + + private: + // The virtual register this range is for, or nullptr if this does not have a + // virtual register (for example, it is in the callRanges bundle). + VirtualRegister* vreg_; + + // The bundle containing this range, null if liveness information is being + // constructed and we haven't started allocating bundles yet. + LiveBundle* bundle_; + + // The code positions in this range. + Range range_; + + // All uses of the virtual register in this range, ordered by location. + InlineForwardList uses_; + + // Total spill weight that calculate from all the uses' policy. Because the + // use's policy can't be changed after initialization, we can update the + // weight whenever a use is added to or remove from this range. This way, we + // don't need to iterate all the uses every time computeSpillWeight() is + // called. + size_t usesSpillWeight_; + + // Number of uses that have policy LUse::FIXED. + uint32_t numFixedUses_; + + // Whether this range contains the virtual register's definition. + bool hasDefinition_; + + LiveRange(VirtualRegister* vreg, Range range) + : vreg_(vreg), + bundle_(nullptr), + range_(range), + usesSpillWeight_(0), + numFixedUses_(0), + hasDefinition_(false) + + { + MOZ_ASSERT(!range.empty()); + } + + void noteAddedUse(UsePosition* use); + void noteRemovedUse(UsePosition* use); + + public: + static LiveRange* FallibleNew(TempAllocator& alloc, VirtualRegister* vreg, + CodePosition from, CodePosition to) { + return new (alloc.fallible()) LiveRange(vreg, Range(from, to)); + } + + VirtualRegister& vreg() const { + MOZ_ASSERT(hasVreg()); + return *vreg_; + } + bool hasVreg() const { return vreg_ != nullptr; } + + LiveBundle* bundle() const { return bundle_; } + + CodePosition from() const { return range_.from; } + CodePosition to() const { return range_.to; } + bool covers(CodePosition pos) const { return pos >= from() && pos < to(); } + + // Whether this range wholly contains other. + bool contains(LiveRange* other) const; + + // Intersect this range with other, returning the subranges of this + // that are before, inside, or after other. + void intersect(LiveRange* other, Range* pre, Range* inside, + Range* post) const; + + // Whether this range has any intersection with other. + bool intersects(LiveRange* other) const; + + UsePositionIterator usesBegin() const { return uses_.begin(); } + UsePosition* lastUse() const { return uses_.back(); } + bool hasUses() const { return !!usesBegin(); } + UsePosition* popUse(); + + bool hasDefinition() const { return hasDefinition_; } + + void setFrom(CodePosition from) { + range_.from = from; + MOZ_ASSERT(!range_.empty()); + } + void setTo(CodePosition to) { + range_.to = to; + MOZ_ASSERT(!range_.empty()); + } + + void setBundle(LiveBundle* bundle) { bundle_ = bundle; } + + void addUse(UsePosition* use); + void tryToMoveDefAndUsesInto(LiveRange* other); + + void setHasDefinition() { + MOZ_ASSERT(!hasDefinition_); + hasDefinition_ = true; + } + + size_t usesSpillWeight() { return usesSpillWeight_; } + uint32_t numFixedUses() { return numFixedUses_; } + +#ifdef JS_JITSPEW + // Return a string describing this range. + UniqueChars toString() const; +#endif + + // Comparator for use in AVL trees. + static int compare(LiveRange* v0, LiveRange* v1) { + // The denoted range includes 'from' but excludes 'to'. + if (v0->to() <= v1->from()) { + return -1; + } + if (v0->from() >= v1->to()) { + return 1; + } + return 0; + } +}; + +// LiveRangePlus is a simple wrapper around a LiveRange*. It caches the +// LiveRange*'s `.range_.from` and `.range_.to` CodePositions. The only +// purpose of this is to avoid some cache misses that would otherwise occur +// when comparing those fields in an AvlTree. This measurably +// speeds up the allocator in some cases. See bug 1814204. + +class LiveRangePlus { + // The LiveRange we're wrapping. + LiveRange* liveRange_; + // Cached versions of liveRange_->range_.from and lr->range_.to + CodePosition from_; + CodePosition to_; + + public: + explicit LiveRangePlus(LiveRange* lr) + : liveRange_(lr), from_(lr->from()), to_(lr->to()) {} + LiveRangePlus() : liveRange_(nullptr) {} + ~LiveRangePlus() { + MOZ_ASSERT(liveRange_ ? from_ == liveRange_->from() + : from_ == CodePosition()); + MOZ_ASSERT(liveRange_ ? to_ == liveRange_->to() : to_ == CodePosition()); + } + + LiveRange* liveRange() const { return liveRange_; } + + // Comparator for use in AVL trees. + static int compare(const LiveRangePlus& lrp0, const LiveRangePlus& lrp1) { + // The denoted range includes 'from' but excludes 'to'. + if (lrp0.to_ <= lrp1.from_) { + return -1; + } + if (lrp0.from_ >= lrp1.to_) { + return 1; + } + return 0; + } +}; + +// Make sure there's no alignment holes or vtable present. Per bug 1814204, +// it's important that this structure is as small as possible. +static_assert(sizeof(LiveRangePlus) == + sizeof(LiveRange*) + 2 * sizeof(CodePosition)); + +// Tracks information about bundles that should all be spilled to the same +// physical location. At the beginning of allocation, each bundle has its own +// spill set. As bundles are split, the new smaller bundles continue to use the +// same spill set. +class SpillSet : public TempObject { + // All bundles with this spill set which have been spilled. All bundles in + // this list will be given the same physical slot. + Vector list_; + + explicit SpillSet(TempAllocator& alloc) : list_(alloc) {} + + public: + static SpillSet* New(TempAllocator& alloc) { + return new (alloc) SpillSet(alloc); + } + + [[nodiscard]] bool addSpilledBundle(LiveBundle* bundle) { + return list_.append(bundle); + } + size_t numSpilledBundles() const { return list_.length(); } + LiveBundle* spilledBundle(size_t i) const { return list_[i]; } + + void setAllocation(LAllocation alloc); +}; + +#ifdef JS_JITSPEW +// See comment on LiveBundle::debugId_ just below. This needs to be atomic +// because TSan automation runs on debug builds will otherwise (correctly) +// report a race. +static mozilla::Atomic LiveBundle_debugIdCounter = + mozilla::Atomic{0}; +#endif + +// A set of live ranges which are all pairwise disjoint. The register allocator +// attempts to find allocations for an entire bundle, and if it fails the +// bundle will be broken into smaller ones which are allocated independently. +class LiveBundle : public TempObject { + // Set to use if this bundle or one it is split into is spilled. + SpillSet* spill_; + + // All the ranges in this set, ordered by location. + InlineForwardList ranges_; + + // Allocation to use for ranges in this set, bogus if unallocated or spilled + // and not yet given a physical stack slot. + LAllocation alloc_; + + // Bundle which entirely contains this one and has no register uses. This + // may or may not be spilled by the allocator, but it can be spilled and + // will not be split. + LiveBundle* spillParent_; + +#ifdef JS_JITSPEW + // This is used only for debug-printing bundles. It gives them an + // identifiable identity in the debug output, which they otherwise wouldn't + // have. + uint32_t debugId_; +#endif + + LiveBundle(SpillSet* spill, LiveBundle* spillParent) + : spill_(spill), spillParent_(spillParent) { +#ifdef JS_JITSPEW + debugId_ = LiveBundle_debugIdCounter++; +#endif + } + + public: + static LiveBundle* FallibleNew(TempAllocator& alloc, SpillSet* spill, + LiveBundle* spillParent) { + return new (alloc.fallible()) LiveBundle(spill, spillParent); + } + + SpillSet* spillSet() const { return spill_; } + void setSpillSet(SpillSet* spill) { spill_ = spill; } + + LiveRange::BundleLinkIterator rangesBegin() const { return ranges_.begin(); } + bool hasRanges() const { return !!rangesBegin(); } + LiveRange* firstRange() const { return LiveRange::get(*rangesBegin()); } + LiveRange* lastRange() const { return LiveRange::get(ranges_.back()); } + LiveRange* rangeFor(CodePosition pos) const; + void removeRange(LiveRange* range); + void removeRangeAndIncrementIterator(LiveRange::BundleLinkIterator& iter) { + ranges_.removeAndIncrement(iter); + } + void addRange(LiveRange* range); + [[nodiscard]] bool addRange(TempAllocator& alloc, VirtualRegister* vreg, + CodePosition from, CodePosition to); + [[nodiscard]] bool addRangeAndDistributeUses(TempAllocator& alloc, + LiveRange* oldRange, + CodePosition from, + CodePosition to); + LiveRange* popFirstRange(); +#ifdef DEBUG + size_t numRanges() const; +#endif + + LAllocation allocation() const { return alloc_; } + void setAllocation(LAllocation alloc) { alloc_ = alloc; } + + LiveBundle* spillParent() const { return spillParent_; } + +#ifdef JS_JITSPEW + uint32_t debugId() const { return debugId_; } + + // Return a string describing this bundle. + UniqueChars toString() const; +#endif +}; + +// Information about the allocation for a virtual register. +class VirtualRegister { + // Instruction which defines this register. + LNode* ins_ = nullptr; + + // Definition in the instruction for this register. + LDefinition* def_ = nullptr; + + // All live ranges for this register. These may overlap each other, and are + // ordered by their start position. + InlineForwardList ranges_; + + // Whether def_ is a temp or an output. + bool isTemp_ = false; + + // Whether this vreg is an input for some phi. This use is not reflected in + // any range on the vreg. + bool usedByPhi_ = false; + + // If this register's definition is MUST_REUSE_INPUT, whether a copy must + // be introduced before the definition that relaxes the policy. + bool mustCopyInput_ = false; + + void operator=(const VirtualRegister&) = delete; + VirtualRegister(const VirtualRegister&) = delete; + + public: + VirtualRegister() = default; + + void init(LNode* ins, LDefinition* def, bool isTemp) { + MOZ_ASSERT(!ins_); + ins_ = ins; + def_ = def; + isTemp_ = isTemp; + } + + LNode* ins() const { return ins_; } + LDefinition* def() const { return def_; } + LDefinition::Type type() const { return def()->type(); } + uint32_t vreg() const { return def()->virtualRegister(); } + bool isCompatible(const AnyRegister& r) const { + return def_->isCompatibleReg(r); + } + bool isCompatible(const VirtualRegister& vr) const { + return def_->isCompatibleDef(*vr.def_); + } + bool isTemp() const { return isTemp_; } + + void setUsedByPhi() { usedByPhi_ = true; } + bool usedByPhi() { return usedByPhi_; } + + void setMustCopyInput() { mustCopyInput_ = true; } + bool mustCopyInput() { return mustCopyInput_; } + + LiveRange::RegisterLinkIterator rangesBegin() const { + return ranges_.begin(); + } + LiveRange::RegisterLinkIterator rangesBegin(LiveRange* range) const { + return ranges_.begin(&range->registerLink); + } + bool hasRanges() const { return !!rangesBegin(); } + LiveRange* firstRange() const { return LiveRange::get(*rangesBegin()); } + LiveRange* lastRange() const { return LiveRange::get(ranges_.back()); } + LiveRange* rangeFor(CodePosition pos, bool preferRegister = false) const; + void removeRange(LiveRange* range); + void addRange(LiveRange* range); + + void removeRangeAndIncrement(LiveRange::RegisterLinkIterator& iter) { + ranges_.removeAndIncrement(iter); + } + + LiveBundle* firstBundle() const { return firstRange()->bundle(); } + + [[nodiscard]] bool addInitialRange(TempAllocator& alloc, CodePosition from, + CodePosition to, size_t* numRanges); + void addInitialUse(UsePosition* use); + void setInitialDefinition(CodePosition from); +}; + +// A sequence of code positions, for tellings BacktrackingAllocator::splitAt +// where to split. +using SplitPositionVector = js::Vector; + +class BacktrackingAllocator : protected RegisterAllocator { + friend class JSONSpewer; + + // This flag is set when testing new allocator modifications. + bool testbed; + + BitSet* liveIn; + FixedList vregs; + + // Allocation state. + StackSlotAllocator stackSlotAllocator; + + // Priority queue element: a bundle and the associated priority. + struct QueueItem { + LiveBundle* bundle; + + QueueItem(LiveBundle* bundle, size_t priority) + : bundle(bundle), priority_(priority) {} + + static size_t priority(const QueueItem& v) { return v.priority_; } + + private: + size_t priority_; + }; + + PriorityQueue allocationQueue; + + // This is a set of LiveRange. They must be non-overlapping. Attempts + // to add an overlapping range will cause AvlTree::insert to MOZ_CRASH(). + using LiveRangeSet = AvlTree; + + // The same, but for LiveRangePlus. See comments on LiveRangePlus. + using LiveRangePlusSet = AvlTree; + + // Each physical register is associated with the set of ranges over which + // that register is currently allocated. + struct PhysicalRegister { + bool allocatable; + AnyRegister reg; + LiveRangePlusSet allocations; + + PhysicalRegister() : allocatable(false) {} + }; + mozilla::Array registers; + + // Ranges of code which are considered to be hot, for which good allocation + // should be prioritized. + LiveRangeSet hotcode; + + struct CallRange : public TempObject, public InlineListNode { + LiveRange::Range range; + + CallRange(CodePosition from, CodePosition to) : range(from, to) {} + + // Comparator for use in AVL trees. + static int compare(CallRange* v0, CallRange* v1) { + if (v0->range.to <= v1->range.from) { + return -1; + } + if (v0->range.from >= v1->range.to) { + return 1; + } + return 0; + } + }; + + // Ranges where all registers must be spilled due to call instructions. + using CallRangeList = InlineList; + CallRangeList callRangesList; + AvlTree callRanges; + + // Information about an allocated stack slot. + struct SpillSlot : public TempObject, + public InlineForwardListNode { + LStackSlot alloc; + LiveRangePlusSet allocated; + + SpillSlot(uint32_t slot, LifoAlloc* alloc) + : alloc(slot), allocated(alloc) {} + }; + using SpillSlotList = InlineForwardList; + + // All allocated slots of each width. + SpillSlotList normalSlots, doubleSlots, quadSlots; + + Vector spilledBundles; + + using LiveBundleVector = Vector; + + // Misc accessors + bool compilingWasm() { return mir->outerInfo().compilingWasm(); } + VirtualRegister& vreg(const LDefinition* def) { + return vregs[def->virtualRegister()]; + } + VirtualRegister& vreg(const LAllocation* alloc) { + MOZ_ASSERT(alloc->isUse()); + return vregs[alloc->toUse()->virtualRegister()]; + } + + // Helpers for creating and adding MoveGroups + [[nodiscard]] bool addMove(LMoveGroup* moves, LiveRange* from, LiveRange* to, + LDefinition::Type type) { + LAllocation fromAlloc = from->bundle()->allocation(); + LAllocation toAlloc = to->bundle()->allocation(); + MOZ_ASSERT(fromAlloc != toAlloc); + return moves->add(fromAlloc, toAlloc, type); + } + + [[nodiscard]] bool moveInput(LInstruction* ins, LiveRange* from, + LiveRange* to, LDefinition::Type type) { + if (from->bundle()->allocation() == to->bundle()->allocation()) { + return true; + } + LMoveGroup* moves = getInputMoveGroup(ins); + return addMove(moves, from, to, type); + } + + [[nodiscard]] bool moveAfter(LInstruction* ins, LiveRange* from, + LiveRange* to, LDefinition::Type type) { + if (from->bundle()->allocation() == to->bundle()->allocation()) { + return true; + } + LMoveGroup* moves = getMoveGroupAfter(ins); + return addMove(moves, from, to, type); + } + + [[nodiscard]] bool moveAtExit(LBlock* block, LiveRange* from, LiveRange* to, + LDefinition::Type type) { + if (from->bundle()->allocation() == to->bundle()->allocation()) { + return true; + } + LMoveGroup* moves = block->getExitMoveGroup(alloc()); + return addMove(moves, from, to, type); + } + + [[nodiscard]] bool moveAtEntry(LBlock* block, LiveRange* from, LiveRange* to, + LDefinition::Type type) { + if (from->bundle()->allocation() == to->bundle()->allocation()) { + return true; + } + LMoveGroup* moves = block->getEntryMoveGroup(alloc()); + return addMove(moves, from, to, type); + } + + // Out-of-line methods, in the same sequence as in BacktrackingAllocator.cpp. + + // Misc helpers: queries about uses + bool isReusedInput(LUse* use, LNode* ins, bool considerCopy); + bool isRegisterUse(UsePosition* use, LNode* ins, bool considerCopy = false); + bool isRegisterDefinition(LiveRange* range); + + // Misc helpers: atomic LIR groups + // (these are all in the parent class, RegisterAllocator) + + // Misc helpers: computation of bundle priorities and spill weights + size_t computePriority(LiveBundle* bundle); + bool minimalDef(LiveRange* range, LNode* ins); + bool minimalUse(LiveRange* range, UsePosition* use); + bool minimalBundle(LiveBundle* bundle, bool* pfixed = nullptr); + size_t computeSpillWeight(LiveBundle* bundle); + size_t maximumSpillWeight(const LiveBundleVector& bundles); + + // Initialization of the allocator + [[nodiscard]] bool init(); + + // Liveness analysis + [[nodiscard]] bool addInitialFixedRange(AnyRegister reg, CodePosition from, + CodePosition to); + [[nodiscard]] bool buildLivenessInfo(); + + // Merging and queueing of LiveRange groups + [[nodiscard]] bool tryMergeBundles(LiveBundle* bundle0, LiveBundle* bundle1); + void allocateStackDefinition(VirtualRegister& reg); + [[nodiscard]] bool tryMergeReusedRegister(VirtualRegister& def, + VirtualRegister& input); + [[nodiscard]] bool mergeAndQueueRegisters(); + + // Implementation of splitting decisions, but not the making of those + // decisions + [[nodiscard]] bool updateVirtualRegisterListsThenRequeueBundles( + LiveBundle* bundle, const LiveBundleVector& newBundles); + + // Implementation of splitting decisions, but not the making of those + // decisions + [[nodiscard]] bool splitAt(LiveBundle* bundle, + const SplitPositionVector& splitPositions); + + // Creation of splitting decisions, but not their implementation + [[nodiscard]] bool splitAcrossCalls(LiveBundle* bundle); + [[nodiscard]] bool trySplitAcrossHotcode(LiveBundle* bundle, bool* success); + [[nodiscard]] bool trySplitAfterLastRegisterUse(LiveBundle* bundle, + LiveBundle* conflict, + bool* success); + [[nodiscard]] bool trySplitBeforeFirstRegisterUse(LiveBundle* bundle, + LiveBundle* conflict, + bool* success); + + // The top level driver for the splitting machinery + [[nodiscard]] bool chooseBundleSplit(LiveBundle* bundle, bool fixed, + LiveBundle* conflict); + + // Bundle allocation + [[nodiscard]] bool computeRequirement(LiveBundle* bundle, + Requirement* prequirement, + Requirement* phint); + [[nodiscard]] bool tryAllocateRegister(PhysicalRegister& r, + LiveBundle* bundle, bool* success, + bool* pfixed, + LiveBundleVector& conflicting); + [[nodiscard]] bool tryAllocateAnyRegister(LiveBundle* bundle, bool* success, + bool* pfixed, + LiveBundleVector& conflicting); + [[nodiscard]] bool evictBundle(LiveBundle* bundle); + [[nodiscard]] bool tryAllocateFixed(LiveBundle* bundle, + Requirement requirement, bool* success, + bool* pfixed, + LiveBundleVector& conflicting); + [[nodiscard]] bool tryAllocateNonFixed(LiveBundle* bundle, + Requirement requirement, + Requirement hint, bool* success, + bool* pfixed, + LiveBundleVector& conflicting); + [[nodiscard]] bool processBundle(MIRGenerator* mir, LiveBundle* bundle); + [[nodiscard]] bool spill(LiveBundle* bundle); + [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool + tryAllocatingRegistersForSpillBundles(); + + // Rewriting of the LIR after bundle processing is done + [[nodiscard]] bool insertAllRanges(LiveRangePlusSet& set, LiveBundle* bundle); + [[nodiscard]] bool pickStackSlot(SpillSet* spill); + [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool pickStackSlots(); + [[nodiscard]] bool moveAtEdge(LBlock* predecessor, LBlock* successor, + LiveRange* from, LiveRange* to, + LDefinition::Type type); + [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool deadRange(LiveRange* range); + [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool + createMoveGroupsFromLiveRangeTransitions(); + size_t findFirstNonCallSafepoint(CodePosition from); + void addLiveRegistersForRange(VirtualRegister& reg, LiveRange* range); + [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool installAllocationsInLIR(); + size_t findFirstSafepoint(CodePosition pos, size_t startFrom); + [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool populateSafepoints(); + [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool annotateMoveGroups(); + + // Debug-printing support +#ifdef JS_JITSPEW + void dumpLiveRangesByVReg(const char* who); + void dumpLiveRangesByBundle(const char* who); + void dumpAllocations(); +#endif + + // Top level of the register allocation machinery, and the only externally + // visible bit. + public: + BacktrackingAllocator(MIRGenerator* mir, LIRGenerator* lir, LIRGraph& graph, + bool testbed) + : RegisterAllocator(mir, lir, graph), + testbed(testbed), + liveIn(nullptr), + callRanges(nullptr) {} + + [[nodiscard]] bool go(); +}; + +} // namespace jit +} // namespace js + +#endif /* jit_BacktrackingAllocator_h */ diff --git a/js/src/jit/Bailouts.cpp b/js/src/jit/Bailouts.cpp new file mode 100644 index 0000000000..3730d8997a --- /dev/null +++ b/js/src/jit/Bailouts.cpp @@ -0,0 +1,352 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/Bailouts.h" + +#include "mozilla/ArrayUtils.h" +#include "mozilla/ScopeExit.h" + +#include "gc/GC.h" +#include "jit/Assembler.h" // jit::FramePointer +#include "jit/BaselineJIT.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/JitSpewer.h" +#include "jit/JSJitFrameIter.h" +#include "jit/SafepointIndex.h" +#include "jit/ScriptFromCalleeToken.h" +#include "vm/Interpreter.h" +#include "vm/JSContext.h" +#include "vm/Stack.h" + +#include "vm/JSScript-inl.h" +#include "vm/Probes-inl.h" +#include "vm/Stack-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::IsInRange; + +#if defined(_WIN32) +# pragma pack(push, 1) +#endif +class js::jit::BailoutStack { + RegisterDump::FPUArray fpregs_; + RegisterDump::GPRArray regs_; + uintptr_t frameSize_; + uintptr_t snapshotOffset_; + + public: + MachineState machineState() { + return MachineState::FromBailout(regs_, fpregs_); + } + uint32_t snapshotOffset() const { return snapshotOffset_; } + uint32_t frameSize() const { return frameSize_; } + uint8_t* parentStackPointer() { + return (uint8_t*)this + sizeof(BailoutStack); + } +}; +#if defined(_WIN32) +# pragma pack(pop) +#endif + +// Make sure the compiler doesn't add extra padding on 32-bit platforms. +static_assert((sizeof(BailoutStack) % 8) == 0, + "BailoutStack should be 8-byte aligned."); + +BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator& activations, + BailoutStack* bailout) + : machine_(bailout->machineState()), activation_(nullptr) { + uint8_t* sp = bailout->parentStackPointer(); + framePointer_ = sp + bailout->frameSize(); + MOZ_RELEASE_ASSERT(uintptr_t(framePointer_) == machine_.read(FramePointer)); + + JSScript* script = + ScriptFromCalleeToken(((JitFrameLayout*)framePointer_)->calleeToken()); + topIonScript_ = script->ionScript(); + + attachOnJitActivation(activations); + snapshotOffset_ = bailout->snapshotOffset(); +} + +BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator& activations, + InvalidationBailoutStack* bailout) + : machine_(bailout->machine()), activation_(nullptr) { + framePointer_ = (uint8_t*)bailout->fp(); + MOZ_RELEASE_ASSERT(uintptr_t(framePointer_) == machine_.read(FramePointer)); + + topIonScript_ = bailout->ionScript(); + attachOnJitActivation(activations); + + uint8_t* returnAddressToFp_ = bailout->osiPointReturnAddress(); + const OsiIndex* osiIndex = topIonScript_->getOsiIndex(returnAddressToFp_); + snapshotOffset_ = osiIndex->snapshotOffset(); +} + +BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator& activations, + const JSJitFrameIter& frame) + : machine_(frame.machineState()) { + framePointer_ = (uint8_t*)frame.fp(); + topIonScript_ = frame.ionScript(); + attachOnJitActivation(activations); + + const OsiIndex* osiIndex = frame.osiIndex(); + snapshotOffset_ = osiIndex->snapshotOffset(); +} + +// This address is a magic number made to cause crashes while indicating that we +// are making an attempt to mark the stack during a bailout. +static constexpr uint32_t FAKE_EXITFP_FOR_BAILOUT_ADDR = 0xba2; +static uint8_t* const FAKE_EXITFP_FOR_BAILOUT = + reinterpret_cast(FAKE_EXITFP_FOR_BAILOUT_ADDR); + +static_assert(!(FAKE_EXITFP_FOR_BAILOUT_ADDR & wasm::ExitFPTag), + "FAKE_EXITFP_FOR_BAILOUT could be mistaken as a low-bit tagged " + "wasm exit fp"); + +bool jit::Bailout(BailoutStack* sp, BaselineBailoutInfo** bailoutInfo) { + JSContext* cx = TlsContext.get(); + MOZ_ASSERT(bailoutInfo); + + // We don't have an exit frame. + MOZ_ASSERT(IsInRange(FAKE_EXITFP_FOR_BAILOUT, 0, 0x1000) && + IsInRange(FAKE_EXITFP_FOR_BAILOUT + sizeof(CommonFrameLayout), + 0, 0x1000), + "Fake exitfp pointer should be within the first page."); + +#ifdef DEBUG + // Reset the counter when we bailed after MDebugEnterGCUnsafeRegion, but + // before the matching MDebugLeaveGCUnsafeRegion. + // + // NOTE: EnterJit ensures the counter is zero when we enter JIT code. + cx->resetInUnsafeRegion(); +#endif + + cx->activation()->asJit()->setJSExitFP(FAKE_EXITFP_FOR_BAILOUT); + + JitActivationIterator jitActivations(cx); + BailoutFrameInfo bailoutData(jitActivations, sp); + JSJitFrameIter frame(jitActivations->asJit()); + MOZ_ASSERT(!frame.ionScript()->invalidated()); + JitFrameLayout* currentFramePtr = frame.jsFrame(); + + JitSpew(JitSpew_IonBailouts, "Took bailout! Snapshot offset: %u", + frame.snapshotOffset()); + + MOZ_ASSERT(IsBaselineJitEnabled(cx)); + + *bailoutInfo = nullptr; + bool success = + BailoutIonToBaseline(cx, bailoutData.activation(), frame, bailoutInfo, + /*exceptionInfo=*/nullptr, BailoutReason::Normal); + MOZ_ASSERT_IF(success, *bailoutInfo != nullptr); + + if (!success) { + MOZ_ASSERT(cx->isExceptionPending()); + JSScript* script = frame.script(); + probes::ExitScript(cx, script, script->function(), + /* popProfilerFrame = */ false); + } + + // This condition was wrong when we entered this bailout function, but it + // might be true now. A GC might have reclaimed all the Jit code and + // invalidated all frames which are currently on the stack. As we are + // already in a bailout, we could not switch to an invalidation + // bailout. When the code of an IonScript which is on the stack is + // invalidated (see InvalidateActivation), we remove references to it and + // increment the reference counter for each activation that appear on the + // stack. As the bailed frame is one of them, we have to decrement it now. + if (frame.ionScript()->invalidated()) { + frame.ionScript()->decrementInvalidationCount(cx->gcContext()); + } + + // NB: Commentary on how |lastProfilingFrame| is set from bailouts. + // + // Once we return to jitcode, any following frames might get clobbered, + // but the current frame will not (as it will be clobbered "in-place" + // with a baseline frame that will share the same frame prefix). + // However, there may be multiple baseline frames unpacked from this + // single Ion frame, which means we will need to once again reset + // |lastProfilingFrame| to point to the correct unpacked last frame + // in |FinishBailoutToBaseline|. + // + // In the case of error, the jitcode will jump immediately to an + // exception handler, which will unwind the frames and properly set + // the |lastProfilingFrame| to point to the frame being resumed into + // (see |AutoResetLastProfilerFrameOnReturnFromException|). + // + // In both cases, we want to temporarily set the |lastProfilingFrame| + // to the current frame being bailed out, and then fix it up later. + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled( + cx->runtime())) { + cx->jitActivation->setLastProfilingFrame(currentFramePtr); + } + + return success; +} + +bool jit::InvalidationBailout(InvalidationBailoutStack* sp, + BaselineBailoutInfo** bailoutInfo) { + sp->checkInvariants(); + + JSContext* cx = TlsContext.get(); + +#ifdef DEBUG + // Reset the counter when we bailed after MDebugEnterGCUnsafeRegion, but + // before the matching MDebugLeaveGCUnsafeRegion. + // + // NOTE: EnterJit ensures the counter is zero when we enter JIT code. + cx->resetInUnsafeRegion(); +#endif + + // We don't have an exit frame. + cx->activation()->asJit()->setJSExitFP(FAKE_EXITFP_FOR_BAILOUT); + + JitActivationIterator jitActivations(cx); + BailoutFrameInfo bailoutData(jitActivations, sp); + JSJitFrameIter frame(jitActivations->asJit()); + JitFrameLayout* currentFramePtr = frame.jsFrame(); + + JitSpew(JitSpew_IonBailouts, "Took invalidation bailout! Snapshot offset: %u", + frame.snapshotOffset()); + + MOZ_ASSERT(IsBaselineJitEnabled(cx)); + + *bailoutInfo = nullptr; + bool success = BailoutIonToBaseline(cx, bailoutData.activation(), frame, + bailoutInfo, /*exceptionInfo=*/nullptr, + BailoutReason::Invalidate); + MOZ_ASSERT_IF(success, *bailoutInfo != nullptr); + + if (!success) { + MOZ_ASSERT(cx->isExceptionPending()); + + // If the bailout failed, then bailout trampoline will pop the + // current frame and jump straight to exception handling code when + // this function returns. Any Gecko Profiler entry pushed for this + // frame will be silently forgotten. + // + // We call ExitScript here to ensure that if the ionScript had Gecko + // Profiler instrumentation, then the entry for it is popped. + // + // However, if the bailout was during argument check, then a + // pseudostack frame would not have been pushed in the first + // place, so don't pop anything in that case. + JSScript* script = frame.script(); + probes::ExitScript(cx, script, script->function(), + /* popProfilerFrame = */ false); + +#ifdef JS_JITSPEW + JitFrameLayout* layout = frame.jsFrame(); + JitSpew(JitSpew_IonInvalidate, "Bailout failed (Fatal Error)"); + JitSpew(JitSpew_IonInvalidate, " calleeToken %p", + (void*)layout->calleeToken()); + JitSpew(JitSpew_IonInvalidate, " callerFramePtr %p", + layout->callerFramePtr()); + JitSpew(JitSpew_IonInvalidate, " ra %p", (void*)layout->returnAddress()); +#endif + } + + frame.ionScript()->decrementInvalidationCount(cx->gcContext()); + + // Make the frame being bailed out the top profiled frame. + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled( + cx->runtime())) { + cx->jitActivation->setLastProfilingFrame(currentFramePtr); + } + + return success; +} + +bool jit::ExceptionHandlerBailout(JSContext* cx, + const InlineFrameIterator& frame, + ResumeFromException* rfe, + const ExceptionBailoutInfo& excInfo) { + // If we are resuming in a finally block, the exception has already + // been captured. + // We can also be propagating debug mode exceptions without there being an + // actual exception pending. For instance, when we return false from an + // operation callback like a timeout handler. + MOZ_ASSERT_IF( + !cx->isExceptionPending(), + excInfo.isFinally() || excInfo.propagatingIonExceptionForDebugMode()); + + JS::AutoSaveExceptionState savedExc(cx); + + JitActivation* act = cx->activation()->asJit(); + uint8_t* prevExitFP = act->jsExitFP(); + auto restoreExitFP = + mozilla::MakeScopeExit([&]() { act->setJSExitFP(prevExitFP); }); + act->setJSExitFP(FAKE_EXITFP_FOR_BAILOUT); + + gc::AutoSuppressGC suppress(cx); + + JitActivationIterator jitActivations(cx); + BailoutFrameInfo bailoutData(jitActivations, frame.frame()); + JSJitFrameIter frameView(jitActivations->asJit()); + JitFrameLayout* currentFramePtr = frameView.jsFrame(); + + BaselineBailoutInfo* bailoutInfo = nullptr; + bool success = BailoutIonToBaseline(cx, bailoutData.activation(), frameView, + &bailoutInfo, &excInfo, + BailoutReason::ExceptionHandler); + if (success) { + MOZ_ASSERT(bailoutInfo); + + // Overwrite the kind so HandleException after the bailout returns + // false, jumping directly to the exception tail. + if (excInfo.propagatingIonExceptionForDebugMode()) { + bailoutInfo->bailoutKind = + mozilla::Some(BailoutKind::IonExceptionDebugMode); + } else if (excInfo.isFinally()) { + bailoutInfo->bailoutKind = mozilla::Some(BailoutKind::Finally); + } + + rfe->kind = ExceptionResumeKind::Bailout; + rfe->stackPointer = bailoutInfo->incomingStack; + rfe->bailoutInfo = bailoutInfo; + } else { + // Drop the exception that triggered the bailout and instead propagate the + // failure caused by processing the bailout (eg. OOM). + savedExc.drop(); + MOZ_ASSERT(!bailoutInfo); + MOZ_ASSERT(cx->isExceptionPending()); + } + + // Make the frame being bailed out the top profiled frame. + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled( + cx->runtime())) { + cx->jitActivation->setLastProfilingFrame(currentFramePtr); + } + + return success; +} + +// Initialize the NamedLambdaObject and CallObject of the current frame if +// needed. +bool jit::EnsureHasEnvironmentObjects(JSContext* cx, AbstractFramePtr fp) { + // Ion does not compile eval scripts. + MOZ_ASSERT(!fp.isEvalFrame()); + + if (fp.isFunctionFrame() && !fp.hasInitialEnvironment() && + fp.callee()->needsFunctionEnvironmentObjects()) { + if (!fp.initFunctionEnvironmentObjects(cx)) { + return false; + } + } + + return true; +} + +void BailoutFrameInfo::attachOnJitActivation( + const JitActivationIterator& jitActivations) { + MOZ_ASSERT(jitActivations->asJit()->jsExitFP() == FAKE_EXITFP_FOR_BAILOUT); + activation_ = jitActivations->asJit(); + activation_->setBailoutData(this); +} + +BailoutFrameInfo::~BailoutFrameInfo() { activation_->cleanBailoutData(); } diff --git a/js/src/jit/Bailouts.h b/js/src/jit/Bailouts.h new file mode 100644 index 0000000000..759f384019 --- /dev/null +++ b/js/src/jit/Bailouts.h @@ -0,0 +1,227 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_Bailouts_h +#define jit_Bailouts_h + +#include "mozilla/Assertions.h" // MOZ_ASSERT + +#include // size_t +#include // uint8_t, uint32_t + +#include "jstypes.h" + +#include "jit/IonTypes.h" // js::jit::Bailout{Id,Kind}, js::jit::SnapshotOffset +#include "jit/MachineState.h" // js::jit::MachineState +#include "js/TypeDecls.h" // jsbytecode +#include "vm/JSContext.h" // JSContext + +namespace js { + +class AbstractFramePtr; + +namespace jit { + +// [SMDOC] IonMonkey Bailouts +// +// A "bailout" is the process of recovering a baseline interpreter frame from an +// IonFrame. Bailouts are implemented in js::jit::BailoutIonToBaseline, which +// has the following callers: +// +// * js::jit::Bailout - This is used when a guard fails in the Ion code +// itself; for example, an LGuardShape fails or an LAddI overflows. See +// callers of CodeGenerator::bailoutFrom() for more examples. +// +// * js::jit::ExceptionHandlerBailout - Ion doesn't implement `catch` or +// `finally`. If an exception is thrown and would be caught by an Ion frame, +// we bail out instead. +// +// * js::jit::InvalidationBailout - We returned to Ion code that was +// invalidated while it was on the stack. See "OSI" below. Ion code can be +// invalidated for several reasons: when GC evicts Ion code to save memory, +// for example, or when assumptions baked into the jitted code are +// invalidated by the VM. +// +// (Some stack inspection can be done without bailing out, including GC stack +// marking, Error object construction, and Gecko profiler sampling.) +// +// Consider the first case. When an Ion guard fails, we can't continue in +// Ion. There's no IC fallback case coming to save us; we've got a broken +// assumption baked into the code we're running. So we jump to an out-of-line +// code path that's responsible for abandoning Ion execution and resuming in +// the baseline interpreter: the bailout path. +// +// We were in the midst of optimized Ion code, so bits of program state may be +// in registers or spilled to the native stack; values may be unboxed; some +// objects may have been optimized away; thanks to inlining, whole call frames +// may be missing. The bailout path must put all these pieces back together +// into the structure the baseline interpreter expects. +// +// The data structure that makes this possible is called a *snapshot*. +// Snapshots are created during Ion codegen and associated with the IonScript; +// they tell how to recover each value in a BaselineFrame from the current +// machine state at a given point in the Ion JIT code. This is potentially +// different at every place in an Ion script where we might bail out. (See +// Snapshots.h.) +// +// The bailout path performs roughly the following steps: +// +// 1. Push a snapshot index and the frame size to the native stack. +// 2. Spill all registers. +// 3. Call js::jit::Bailout to reconstruct the baseline frame(s). +// 4. memmove() those to the right place on the native stack. +// 5. Jump into the baseline interpreter. +// +// When C++ code invalidates Ion code, we do on-stack invalidation, or OSI, to +// arrange for every affected Ion frame on the stack to bail out as soon as +// control returns to it. OSI patches every instruction in the JIT code that's +// at a return address currently on the stack. See InvalidateActivation. +// +// +// ## Bailout path implementation details +// +// Ion code has a lot of guards, so each bailout path must be small. Steps 2 +// and 3 above are therefore implemented by a shared per-Runtime trampoline, +// rt->jitRuntime()->getGenericBailoutHandler(). +// +// We implement step 1 like this: +// +// _bailout_ID_1: +// push 1 +// jmp _deopt +// _bailout_ID_2: +// push 2 +// jmp _deopt +// ... +// _deopt: +// push imm(FrameSize) +// call _global_bailout_handler + +// BailoutStack is an architecture specific pointer to the stack, given by the +// bailout handler. +class BailoutStack; +class InvalidationBailoutStack; + +class IonScript; +class InlineFrameIterator; +class JitActivation; +class JitActivationIterator; +class JSJitFrameIter; +struct ResumeFromException; + +// Must be implemented by each architecture. + +// This structure is constructed before recovering the baseline frames for a +// bailout. It records all information extracted from the stack, and which are +// needed for the JSJitFrameIter. +class BailoutFrameInfo { + MachineState machine_; + uint8_t* framePointer_; + IonScript* topIonScript_; + uint32_t snapshotOffset_; + JitActivation* activation_; + + void attachOnJitActivation(const JitActivationIterator& activations); + + public: + BailoutFrameInfo(const JitActivationIterator& activations, BailoutStack* sp); + BailoutFrameInfo(const JitActivationIterator& activations, + InvalidationBailoutStack* sp); + BailoutFrameInfo(const JitActivationIterator& activations, + const JSJitFrameIter& frame); + ~BailoutFrameInfo(); + + uint8_t* fp() const { return framePointer_; } + SnapshotOffset snapshotOffset() const { return snapshotOffset_; } + const MachineState* machineState() const { return &machine_; } + IonScript* ionScript() const { return topIonScript_; } + JitActivation* activation() const { return activation_; } +}; + +[[nodiscard]] bool EnsureHasEnvironmentObjects(JSContext* cx, + AbstractFramePtr fp); + +struct BaselineBailoutInfo; + +// Called from a bailout thunk. +[[nodiscard]] bool Bailout(BailoutStack* sp, BaselineBailoutInfo** info); + +// Called from the invalidation thunk. +[[nodiscard]] bool InvalidationBailout(InvalidationBailoutStack* sp, + BaselineBailoutInfo** info); + +class ExceptionBailoutInfo { + size_t frameNo_; + jsbytecode* resumePC_; + size_t numExprSlots_; + bool isFinally_ = false; + RootedValue finallyException_; + bool forcedReturn_; + + public: + ExceptionBailoutInfo(JSContext* cx, size_t frameNo, jsbytecode* resumePC, + size_t numExprSlots) + : frameNo_(frameNo), + resumePC_(resumePC), + numExprSlots_(numExprSlots), + finallyException_(cx), + forcedReturn_(cx->isPropagatingForcedReturn()) {} + + explicit ExceptionBailoutInfo(JSContext* cx) + : frameNo_(0), + resumePC_(nullptr), + numExprSlots_(0), + finallyException_(cx), + forcedReturn_(cx->isPropagatingForcedReturn()) {} + + bool catchingException() const { return !!resumePC_; } + bool propagatingIonExceptionForDebugMode() const { return !resumePC_; } + + size_t frameNo() const { + MOZ_ASSERT(catchingException()); + return frameNo_; + } + jsbytecode* resumePC() const { + MOZ_ASSERT(catchingException()); + return resumePC_; + } + size_t numExprSlots() const { + MOZ_ASSERT(catchingException()); + return numExprSlots_; + } + + bool isFinally() const { return isFinally_; } + void setFinallyException(JS::Value& exception) { + MOZ_ASSERT(!isFinally()); + isFinally_ = true; + finallyException_ = exception; + } + HandleValue finallyException() const { + MOZ_ASSERT(isFinally()); + return finallyException_; + } + + bool forcedReturn() const { return forcedReturn_; } +}; + +// Called from the exception handler to enter a catch or finally block. +[[nodiscard]] bool ExceptionHandlerBailout(JSContext* cx, + const InlineFrameIterator& frame, + ResumeFromException* rfe, + const ExceptionBailoutInfo& excInfo); + +[[nodiscard]] bool FinishBailoutToBaseline(BaselineBailoutInfo* bailoutInfoArg); + +#ifdef DEBUG +[[nodiscard]] bool AssertBailoutStackDepth(JSContext* cx, JSScript* script, + jsbytecode* pc, ResumeMode mode, + uint32_t exprStackSlots); +#endif + +} // namespace jit +} // namespace js + +#endif /* jit_Bailouts_h */ diff --git a/js/src/jit/BaselineBailouts.cpp b/js/src/jit/BaselineBailouts.cpp new file mode 100644 index 0000000000..c82a05d0ea --- /dev/null +++ b/js/src/jit/BaselineBailouts.cpp @@ -0,0 +1,2092 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/Assertions.h" +#include "mozilla/ScopeExit.h" + +#include "builtin/ModuleObject.h" +#include "debugger/DebugAPI.h" +#include "gc/GC.h" +#include "jit/Bailouts.h" +#include "jit/BaselineFrame.h" +#include "jit/BaselineIC.h" +#include "jit/BaselineJIT.h" +#include "jit/CalleeToken.h" +#include "jit/Invalidation.h" +#include "jit/Ion.h" +#include "jit/IonScript.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/JitSpewer.h" +#include "jit/JitZone.h" +#include "jit/RematerializedFrame.h" +#include "jit/SharedICRegisters.h" +#include "jit/Simulator.h" +#include "js/friend/StackLimits.h" // js::AutoCheckRecursionLimit, js::ReportOverRecursed +#include "js/Utility.h" +#include "util/Memory.h" +#include "vm/ArgumentsObject.h" +#include "vm/BytecodeUtil.h" +#include "vm/JitActivation.h" + +#include "jit/JitFrames-inl.h" +#include "vm/JSContext-inl.h" +#include "vm/JSScript-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::DebugOnly; +using mozilla::Maybe; + +// BaselineStackBuilder may reallocate its buffer if the current one is too +// small. To avoid dangling pointers, BufferPointer represents a pointer into +// this buffer as a pointer to the header and a fixed offset. +template +class BufferPointer { + const UniquePtr& header_; + size_t offset_; + bool heap_; + + public: + BufferPointer(const UniquePtr& header, size_t offset, + bool heap) + : header_(header), offset_(offset), heap_(heap) {} + + T* get() const { + BaselineBailoutInfo* header = header_.get(); + if (!heap_) { + return (T*)(header->incomingStack + offset_); + } + + uint8_t* p = header->copyStackTop - offset_; + MOZ_ASSERT(p >= header->copyStackBottom && p < header->copyStackTop); + return (T*)p; + } + + void set(const T& value) { *get() = value; } + + // Note: we return a copy instead of a reference, to avoid potential memory + // safety hazards when the underlying buffer gets resized. + const T operator*() const { return *get(); } + T* operator->() const { return get(); } +}; + +/** + * BaselineStackBuilder helps abstract the process of rebuilding the C stack on + * the heap. It takes a bailout iterator and keeps track of the point on the C + * stack from which the reconstructed frames will be written. + * + * It exposes methods to write data into the heap memory storing the + * reconstructed stack. It also exposes method to easily calculate addresses. + * This includes both the virtual address that a particular value will be at + * when it's eventually copied onto the stack, as well as the current actual + * address of that value (whether on the heap allocated portion being + * constructed or the existing stack). + * + * The abstraction handles transparent re-allocation of the heap memory when it + * needs to be enlarged to accommodate new data. Similarly to the C stack, the + * data that's written to the reconstructed stack grows from high to low in + * memory. + * + * The lowest region of the allocated memory contains a BaselineBailoutInfo + * structure that points to the start and end of the written data. + */ +class MOZ_STACK_CLASS BaselineStackBuilder { + JSContext* cx_; + JitFrameLayout* frame_ = nullptr; + SnapshotIterator& iter_; + RootedValueVector outermostFrameFormals_; + + size_t bufferTotal_ = 1024; + size_t bufferAvail_ = 0; + size_t bufferUsed_ = 0; + size_t framePushed_ = 0; + + UniquePtr header_; + + JSScript* script_; + JSFunction* fun_; + const ExceptionBailoutInfo* excInfo_; + ICScript* icScript_; + + jsbytecode* pc_ = nullptr; + JSOp op_ = JSOp::Nop; + mozilla::Maybe resumeMode_; + uint32_t exprStackSlots_ = 0; + void* prevFramePtr_ = nullptr; + Maybe> blFrame_; + + size_t frameNo_ = 0; + JSFunction* nextCallee_ = nullptr; + + BailoutKind bailoutKind_; + + // The baseline frames we will reconstruct on the heap are not + // rooted, so GC must be suppressed. + gc::AutoSuppressGC suppress_; + + public: + BaselineStackBuilder(JSContext* cx, const JSJitFrameIter& frameIter, + SnapshotIterator& iter, + const ExceptionBailoutInfo* excInfo, + BailoutReason reason); + + [[nodiscard]] bool init() { + MOZ_ASSERT(!header_); + MOZ_ASSERT(bufferUsed_ == 0); + + uint8_t* bufferRaw = cx_->pod_calloc(bufferTotal_); + if (!bufferRaw) { + return false; + } + bufferAvail_ = bufferTotal_ - sizeof(BaselineBailoutInfo); + + header_.reset(new (bufferRaw) BaselineBailoutInfo()); + header_->incomingStack = reinterpret_cast(frame_); + header_->copyStackTop = bufferRaw + bufferTotal_; + header_->copyStackBottom = header_->copyStackTop; + return true; + } + + [[nodiscard]] bool buildOneFrame(); + bool done(); + void nextFrame(); + + JSScript* script() const { return script_; } + size_t frameNo() const { return frameNo_; } + bool isOutermostFrame() const { return frameNo_ == 0; } + MutableHandleValueVector outermostFrameFormals() { + return &outermostFrameFormals_; + } + BailoutKind bailoutKind() const { return bailoutKind_; } + + inline JitFrameLayout* startFrame() { return frame_; } + + BaselineBailoutInfo* info() { + MOZ_ASSERT(header_); + return header_.get(); + } + + BaselineBailoutInfo* takeBuffer() { + MOZ_ASSERT(header_); + return header_.release(); + } + + private: + [[nodiscard]] bool initFrame(); + [[nodiscard]] bool buildBaselineFrame(); + [[nodiscard]] bool buildArguments(); + [[nodiscard]] bool buildFixedSlots(); + [[nodiscard]] bool fixUpCallerArgs(MutableHandleValueVector savedCallerArgs, + bool* fixedUp); + [[nodiscard]] bool buildFinallyException(); + [[nodiscard]] bool buildExpressionStack(); + [[nodiscard]] bool finishLastFrame(); + + [[nodiscard]] bool prepareForNextFrame(HandleValueVector savedCallerArgs); + [[nodiscard]] bool finishOuterFrame(); + [[nodiscard]] bool buildStubFrame(uint32_t frameSize, + HandleValueVector savedCallerArgs); + [[nodiscard]] bool buildRectifierFrame(uint32_t actualArgc, + size_t endOfBaselineStubArgs); + +#ifdef DEBUG + [[nodiscard]] bool validateFrame(); +#endif + +#ifdef DEBUG + bool envChainSlotCanBeOptimized(); +#endif + + bool isPrologueBailout(); + jsbytecode* getResumePC(); + void* getStubReturnAddress(); + + uint32_t exprStackSlots() const { return exprStackSlots_; } + + // Returns true if we're bailing out to a catch or finally block in this frame + bool catchingException() const { + return excInfo_ && excInfo_->catchingException() && + excInfo_->frameNo() == frameNo_; + } + + // Returns true if we're bailing out to a finally block in this frame. + bool resumingInFinallyBlock() const { + return catchingException() && excInfo_->isFinally(); + } + + bool forcedReturn() const { return excInfo_ && excInfo_->forcedReturn(); } + + // Returns true if we're bailing out in place for debug mode + bool propagatingIonExceptionForDebugMode() const { + return excInfo_ && excInfo_->propagatingIonExceptionForDebugMode(); + } + + void* prevFramePtr() const { + MOZ_ASSERT(prevFramePtr_); + return prevFramePtr_; + } + BufferPointer& blFrame() { return blFrame_.ref(); } + + void setNextCallee(JSFunction* nextCallee, + TrialInliningState trialInliningState); + JSFunction* nextCallee() const { return nextCallee_; } + + jsbytecode* pc() const { return pc_; } + bool resumeAfter() const { + return !catchingException() && iter_.resumeAfter(); + } + + ResumeMode resumeMode() const { return *resumeMode_; } + + bool needToSaveCallerArgs() const { + return resumeMode() == ResumeMode::InlinedAccessor; + } + + [[nodiscard]] bool enlarge() { + MOZ_ASSERT(header_ != nullptr); + if (bufferTotal_ & mozilla::tl::MulOverflowMask<2>::value) { + ReportOutOfMemory(cx_); + return false; + } + + size_t newSize = bufferTotal_ * 2; + uint8_t* newBufferRaw = cx_->pod_calloc(newSize); + if (!newBufferRaw) { + return false; + } + + // Initialize the new buffer. + // + // Before: + // + // [ Header | .. | Payload ] + // + // After: + // + // [ Header | ............... | Payload ] + // + // Size of Payload is |bufferUsed_|. + // + // We need to copy from the old buffer and header to the new buffer before + // we set header_ (this deletes the old buffer). + // + // We also need to update |copyStackBottom| and |copyStackTop| because these + // fields point to the Payload's start and end, respectively. + using BailoutInfoPtr = UniquePtr; + BailoutInfoPtr newHeader(new (newBufferRaw) BaselineBailoutInfo(*header_)); + newHeader->copyStackTop = newBufferRaw + newSize; + newHeader->copyStackBottom = newHeader->copyStackTop - bufferUsed_; + memcpy(newHeader->copyStackBottom, header_->copyStackBottom, bufferUsed_); + bufferTotal_ = newSize; + bufferAvail_ = newSize - (sizeof(BaselineBailoutInfo) + bufferUsed_); + header_ = std::move(newHeader); + return true; + } + + void resetFramePushed() { framePushed_ = 0; } + + size_t framePushed() const { return framePushed_; } + + [[nodiscard]] bool subtract(size_t size, const char* info = nullptr) { + // enlarge the buffer if need be. + while (size > bufferAvail_) { + if (!enlarge()) { + return false; + } + } + + // write out element. + header_->copyStackBottom -= size; + bufferAvail_ -= size; + bufferUsed_ += size; + framePushed_ += size; + if (info) { + JitSpew(JitSpew_BaselineBailouts, " SUB_%03d %p/%p %-15s", + (int)size, header_->copyStackBottom, + virtualPointerAtStackOffset(0), info); + } + return true; + } + + template + [[nodiscard]] bool write(const T& t) { + MOZ_ASSERT(!(uintptr_t(&t) >= uintptr_t(header_->copyStackBottom) && + uintptr_t(&t) < uintptr_t(header_->copyStackTop)), + "Should not reference memory that can be freed"); + if (!subtract(sizeof(T))) { + return false; + } + memcpy(header_->copyStackBottom, &t, sizeof(T)); + return true; + } + + template + [[nodiscard]] bool writePtr(T* t, const char* info) { + if (!write(t)) { + return false; + } + if (info) { + JitSpew(JitSpew_BaselineBailouts, " WRITE_PTR %p/%p %-15s %p", + header_->copyStackBottom, virtualPointerAtStackOffset(0), info, + t); + } + return true; + } + + [[nodiscard]] bool writeWord(size_t w, const char* info) { + if (!write(w)) { + return false; + } + if (info) { + if (sizeof(size_t) == 4) { + JitSpew(JitSpew_BaselineBailouts, " WRITE_WRD %p/%p %-15s %08zx", + header_->copyStackBottom, virtualPointerAtStackOffset(0), info, + w); + } else { + JitSpew(JitSpew_BaselineBailouts, " WRITE_WRD %p/%p %-15s %016zx", + header_->copyStackBottom, virtualPointerAtStackOffset(0), info, + w); + } + } + return true; + } + + [[nodiscard]] bool writeValue(const Value& val, const char* info) { + if (!write(val)) { + return false; + } + if (info) { + JitSpew(JitSpew_BaselineBailouts, + " WRITE_VAL %p/%p %-15s %016" PRIx64, + header_->copyStackBottom, virtualPointerAtStackOffset(0), info, + *((uint64_t*)&val)); + } + return true; + } + + [[nodiscard]] bool maybeWritePadding(size_t alignment, size_t after, + const char* info) { + MOZ_ASSERT(framePushed_ % sizeof(Value) == 0); + MOZ_ASSERT(after % sizeof(Value) == 0); + size_t offset = ComputeByteAlignment(after, alignment); + while (framePushed_ % alignment != offset) { + if (!writeValue(MagicValue(JS_ARG_POISON), info)) { + return false; + } + } + + return true; + } + + void setResumeFramePtr(void* resumeFramePtr) { + header_->resumeFramePtr = resumeFramePtr; + } + + void setResumeAddr(void* resumeAddr) { header_->resumeAddr = resumeAddr; } + + template + BufferPointer pointerAtStackOffset(size_t offset) { + if (offset < bufferUsed_) { + // Calculate offset from copyStackTop. + offset = header_->copyStackTop - (header_->copyStackBottom + offset); + return BufferPointer(header_, offset, /* heap = */ true); + } + + return BufferPointer(header_, offset - bufferUsed_, /* heap = */ false); + } + + BufferPointer valuePointerAtStackOffset(size_t offset) { + return pointerAtStackOffset(offset); + } + + inline uint8_t* virtualPointerAtStackOffset(size_t offset) { + if (offset < bufferUsed_) { + return reinterpret_cast(frame_) - (bufferUsed_ - offset); + } + return reinterpret_cast(frame_) + (offset - bufferUsed_); + } +}; + +BaselineStackBuilder::BaselineStackBuilder(JSContext* cx, + const JSJitFrameIter& frameIter, + SnapshotIterator& iter, + const ExceptionBailoutInfo* excInfo, + BailoutReason reason) + : cx_(cx), + frame_(static_cast(frameIter.current())), + iter_(iter), + outermostFrameFormals_(cx), + script_(frameIter.script()), + fun_(frameIter.maybeCallee()), + excInfo_(excInfo), + icScript_(script_->jitScript()->icScript()), + bailoutKind_(iter.bailoutKind()), + suppress_(cx) { + MOZ_ASSERT(bufferTotal_ >= sizeof(BaselineBailoutInfo)); + if (reason == BailoutReason::Invalidate) { + bailoutKind_ = BailoutKind::OnStackInvalidation; + } +} + +bool BaselineStackBuilder::initFrame() { + // Get the pc and ResumeMode. If we are handling an exception, resume at the + // pc of the catch or finally block. + if (catchingException()) { + pc_ = excInfo_->resumePC(); + resumeMode_ = mozilla::Some(ResumeMode::ResumeAt); + } else { + pc_ = script_->offsetToPC(iter_.pcOffset()); + resumeMode_ = mozilla::Some(iter_.resumeMode()); + } + op_ = JSOp(*pc_); + + // If we are catching an exception, we are bailing out to a catch or + // finally block and this is the frame where we will resume. Usually the + // expression stack should be empty in this case but there can be + // iterators on the stack. + if (catchingException()) { + exprStackSlots_ = excInfo_->numExprSlots(); + } else { + uint32_t totalFrameSlots = iter_.numAllocations(); + uint32_t fixedSlots = script_->nfixed(); + uint32_t argSlots = CountArgSlots(script_, fun_); + uint32_t intermediates = NumIntermediateValues(resumeMode()); + exprStackSlots_ = totalFrameSlots - fixedSlots - argSlots - intermediates; + + // Verify that there was no underflow. + MOZ_ASSERT(exprStackSlots_ <= totalFrameSlots); + } + + JitSpew(JitSpew_BaselineBailouts, " Unpacking %s:%u:%u", + script_->filename(), script_->lineno(), script_->column()); + JitSpew(JitSpew_BaselineBailouts, " [BASELINE-JS FRAME]"); + + // Write the previous frame pointer value. For the outermost frame we reuse + // the value in the JitFrameLayout already on the stack. Record the virtual + // stack offset at this location. Later on, if we end up writing out a + // BaselineStub frame for the next callee, we'll need to save the address. + if (!isOutermostFrame()) { + if (!writePtr(prevFramePtr(), "PrevFramePtr")) { + return false; + } + } + prevFramePtr_ = virtualPointerAtStackOffset(0); + + resetFramePushed(); + + return true; +} + +void BaselineStackBuilder::setNextCallee( + JSFunction* nextCallee, TrialInliningState trialInliningState) { + nextCallee_ = nextCallee; + + if (trialInliningState == TrialInliningState::Inlined) { + // Update icScript_ to point to the icScript of nextCallee + const uint32_t pcOff = script_->pcToOffset(pc_); + icScript_ = icScript_->findInlinedChild(pcOff); + } else { + // If we don't know for certain that it's TrialInliningState::Inlined, + // just use the callee's own ICScript. We could still have the trial + // inlined ICScript available, but we also could not if we transitioned + // to TrialInliningState::Failure after being monomorphic inlined. + icScript_ = nextCallee->nonLazyScript()->jitScript()->icScript(); + } +} + +bool BaselineStackBuilder::done() { + if (!iter_.moreFrames()) { + MOZ_ASSERT(!nextCallee_); + return true; + } + return catchingException(); +} + +void BaselineStackBuilder::nextFrame() { + MOZ_ASSERT(nextCallee_); + fun_ = nextCallee_; + script_ = fun_->nonLazyScript(); + nextCallee_ = nullptr; + + // Scripts with an IonScript must also have a BaselineScript. + MOZ_ASSERT(script_->hasBaselineScript()); + + frameNo_++; + iter_.nextInstruction(); +} + +// Build the BaselineFrame struct +bool BaselineStackBuilder::buildBaselineFrame() { + if (!subtract(BaselineFrame::Size(), "BaselineFrame")) { + return false; + } + blFrame_.reset(); + blFrame_.emplace(pointerAtStackOffset(0)); + + uint32_t flags = BaselineFrame::RUNNING_IN_INTERPRETER; + + // If we are bailing to a script whose execution is observed, mark the + // baseline frame as a debuggee frame. This is to cover the case where we + // don't rematerialize the Ion frame via the Debugger. + if (script_->isDebuggee()) { + flags |= BaselineFrame::DEBUGGEE; + } + + // Get |envChain|. + JSObject* envChain = nullptr; + Value envChainSlot = iter_.read(); + if (envChainSlot.isObject()) { + // The env slot has been updated from UndefinedValue. It must be the + // complete initial environment. + envChain = &envChainSlot.toObject(); + + // Set the HAS_INITIAL_ENV flag if needed. See IsFrameInitialEnvironment. + MOZ_ASSERT(!script_->isForEval()); + if (fun_ && fun_->needsFunctionEnvironmentObjects()) { + MOZ_ASSERT(fun_->nonLazyScript()->initialEnvironmentShape()); + flags |= BaselineFrame::HAS_INITIAL_ENV; + } + } else { + MOZ_ASSERT(envChainSlot.isUndefined() || + envChainSlot.isMagic(JS_OPTIMIZED_OUT)); + MOZ_ASSERT(envChainSlotCanBeOptimized()); + + // The env slot has been optimized out. + // Get it from the function or script. + if (fun_) { + envChain = fun_->environment(); + } else if (script_->isModule()) { + envChain = script_->module()->environment(); + } else { + // For global scripts without a non-syntactic env the env + // chain is the script's global lexical environment. (We do + // not compile scripts with a non-syntactic global scope). + // Also note that it's invalid to resume into the prologue in + // this case because the prologue expects the env chain in R1 + // for eval and global scripts. + MOZ_ASSERT(!script_->isForEval()); + MOZ_ASSERT(!script_->hasNonSyntacticScope()); + envChain = &(script_->global().lexicalEnvironment()); + } + } + + // Write |envChain|. + MOZ_ASSERT(envChain); + JitSpew(JitSpew_BaselineBailouts, " EnvChain=%p", envChain); + blFrame()->setEnvironmentChain(envChain); + + // Get |returnValue| if present. + Value returnValue = UndefinedValue(); + if (script_->noScriptRval()) { + // Don't use the return value (likely a JS_OPTIMIZED_OUT MagicValue) to + // not confuse Baseline. + iter_.skip(); + } else { + returnValue = iter_.read(); + flags |= BaselineFrame::HAS_RVAL; + } + + // Write |returnValue|. + JitSpew(JitSpew_BaselineBailouts, " ReturnValue=%016" PRIx64, + *((uint64_t*)&returnValue)); + blFrame()->setReturnValue(returnValue); + + // Get |argsObj| if present. + ArgumentsObject* argsObj = nullptr; + if (script_->needsArgsObj()) { + Value maybeArgsObj = iter_.read(); + MOZ_ASSERT(maybeArgsObj.isObject() || maybeArgsObj.isUndefined() || + maybeArgsObj.isMagic(JS_OPTIMIZED_OUT)); + if (maybeArgsObj.isObject()) { + argsObj = &maybeArgsObj.toObject().as(); + } + } + + // Note: we do not need to initialize the scratchValue field in BaselineFrame. + + // Write |flags|. + blFrame()->setFlags(flags); + + // Write |icScript|. + JitSpew(JitSpew_BaselineBailouts, " ICScript=%p", icScript_); + blFrame()->setICScript(icScript_); + + // initArgsObjUnchecked modifies the frame's flags, so call it after setFlags. + if (argsObj) { + blFrame()->initArgsObjUnchecked(*argsObj); + } + return true; +} + +// Overwrite the pushed args present in the calling frame with +// the unpacked |thisv| and argument values. +bool BaselineStackBuilder::buildArguments() { + Value thisv = iter_.read(); + JitSpew(JitSpew_BaselineBailouts, " Is function!"); + JitSpew(JitSpew_BaselineBailouts, " thisv=%016" PRIx64, + *((uint64_t*)&thisv)); + + size_t thisvOffset = framePushed() + JitFrameLayout::offsetOfThis(); + valuePointerAtStackOffset(thisvOffset).set(thisv); + + MOZ_ASSERT(iter_.numAllocations() >= CountArgSlots(script_, fun_)); + JitSpew(JitSpew_BaselineBailouts, + " frame slots %u, nargs %zu, nfixed %zu", iter_.numAllocations(), + fun_->nargs(), script_->nfixed()); + + bool shouldStoreOutermostFormals = + isOutermostFrame() && !script_->argsObjAliasesFormals(); + if (shouldStoreOutermostFormals) { + // This is the first (outermost) frame and we don't have an + // arguments object aliasing the formals. Due to UCE and phi + // elimination, we could store an UndefinedValue() here for + // formals we think are unused, but locals may still reference the + // original argument slot (MParameter/LArgument) and expect the + // original Value. To avoid this problem, store the formals in a + // Vector until we are done. + MOZ_ASSERT(outermostFrameFormals().empty()); + if (!outermostFrameFormals().resize(fun_->nargs())) { + return false; + } + } + + for (uint32_t i = 0; i < fun_->nargs(); i++) { + Value arg = iter_.read(); + JitSpew(JitSpew_BaselineBailouts, " arg %d = %016" PRIx64, (int)i, + *((uint64_t*)&arg)); + if (!isOutermostFrame()) { + size_t argOffset = framePushed() + JitFrameLayout::offsetOfActualArg(i); + valuePointerAtStackOffset(argOffset).set(arg); + } else if (shouldStoreOutermostFormals) { + outermostFrameFormals()[i].set(arg); + } else { + // When the arguments object aliases the formal arguments, then + // JSOp::SetArg mutates the argument object. In such cases, the + // list of arguments reported by the snapshot are only aliases + // of argument object slots which are optimized to only store + // differences compared to arguments which are on the stack. + } + } + return true; +} + +bool BaselineStackBuilder::buildFixedSlots() { + for (uint32_t i = 0; i < script_->nfixed(); i++) { + Value slot = iter_.read(); + if (!writeValue(slot, "FixedValue")) { + return false; + } + } + return true; +} + +// The caller side of inlined js::fun_call and accessors must look +// like the function wasn't inlined. +bool BaselineStackBuilder::fixUpCallerArgs( + MutableHandleValueVector savedCallerArgs, bool* fixedUp) { + MOZ_ASSERT(!*fixedUp); + + // Inlining of SpreadCall-like frames not currently supported. + MOZ_ASSERT(!IsSpreadOp(op_)); + + if (resumeMode() != ResumeMode::InlinedFunCall && !needToSaveCallerArgs()) { + return true; + } + + // Calculate how many arguments are consumed by the inlined call. + // All calls pass |callee| and |this|. + uint32_t inlinedArgs = 2; + if (resumeMode() == ResumeMode::InlinedFunCall) { + // The first argument to an inlined FunCall becomes |this|, + // if it exists. The rest are passed normally. + MOZ_ASSERT(IsInvokeOp(op_)); + inlinedArgs += GET_ARGC(pc_) > 0 ? GET_ARGC(pc_) - 1 : 0; + } else { + MOZ_ASSERT(resumeMode() == ResumeMode::InlinedAccessor); + MOZ_ASSERT(IsIonInlinableGetterOrSetterOp(op_)); + // Setters are passed one argument. Getters are passed none. + if (IsSetPropOp(op_)) { + inlinedArgs++; + } + } + + // Calculate how many values are live on the stack across the call, + // and push them. + MOZ_ASSERT(inlinedArgs <= exprStackSlots()); + uint32_t liveStackSlots = exprStackSlots() - inlinedArgs; + + JitSpew(JitSpew_BaselineBailouts, + " pushing %u expression stack slots before fixup", + liveStackSlots); + for (uint32_t i = 0; i < liveStackSlots; i++) { + Value v = iter_.read(); + if (!writeValue(v, "StackValue")) { + return false; + } + } + + // When we inline js::fun_call, we bypass the native and inline the + // target directly. When rebuilding the stack, we need to fill in + // the right number of slots to make it look like the js_native was + // actually called. + if (resumeMode() == ResumeMode::InlinedFunCall) { + // We must transform the stack from |target, this, args| to + // |js_fun_call, target, this, args|. The value of |js_fun_call| + // will never be observed, so we push |undefined| for it, followed + // by the remaining arguments. + JitSpew(JitSpew_BaselineBailouts, + " pushing undefined to fixup funcall"); + if (!writeValue(UndefinedValue(), "StackValue")) { + return false; + } + if (GET_ARGC(pc_) > 0) { + JitSpew(JitSpew_BaselineBailouts, + " pushing %u expression stack slots", inlinedArgs); + for (uint32_t i = 0; i < inlinedArgs; i++) { + Value arg = iter_.read(); + if (!writeValue(arg, "StackValue")) { + return false; + } + } + } else { + // When we inline FunCall with no arguments, we push an extra + // |undefined| value for |this|. That value should not appear + // in the rebuilt baseline frame. + JitSpew(JitSpew_BaselineBailouts, " pushing target of funcall"); + Value target = iter_.read(); + if (!writeValue(target, "StackValue")) { + return false; + } + // Skip |this|. + iter_.skip(); + } + } + + if (needToSaveCallerArgs()) { + // Save the actual arguments. They are needed to rebuild the callee frame. + if (!savedCallerArgs.resize(inlinedArgs)) { + return false; + } + for (uint32_t i = 0; i < inlinedArgs; i++) { + savedCallerArgs[i].set(iter_.read()); + } + + if (IsSetPropOp(op_)) { + // The RHS argument to SetProp remains on the stack after the + // operation and is observable, so we have to fill it in. + Value initialArg = savedCallerArgs[inlinedArgs - 1]; + JitSpew(JitSpew_BaselineBailouts, + " pushing setter's initial argument"); + if (!writeValue(initialArg, "StackValue")) { + return false; + } + } + } + + *fixedUp = true; + return true; +} + +bool BaselineStackBuilder::buildExpressionStack() { + JitSpew(JitSpew_BaselineBailouts, " pushing %u expression stack slots", + exprStackSlots()); + for (uint32_t i = 0; i < exprStackSlots(); i++) { + Value v; + // If we are in the middle of propagating an exception from Ion by + // bailing to baseline due to debug mode, we might not have all + // the stack if we are at the newest frame. + // + // For instance, if calling |f()| pushed an Ion frame which threw, + // the snapshot expects the return value to be pushed, but it's + // possible nothing was pushed before we threw. + // + // We therefore use a fallible read here. + if (!iter_.tryRead(&v)) { + MOZ_ASSERT(propagatingIonExceptionForDebugMode() && !iter_.moreFrames()); + v = MagicValue(JS_OPTIMIZED_OUT); + } + if (!writeValue(v, "StackValue")) { + return false; + } + } + + if (resumeMode() == ResumeMode::ResumeAfterCheckIsObject) { + JitSpew(JitSpew_BaselineBailouts, + " Checking that intermediate value is an object"); + Value returnVal; + if (iter_.tryRead(&returnVal) && !returnVal.isObject()) { + MOZ_ASSERT(!returnVal.isMagic()); + JitSpew(JitSpew_BaselineBailouts, + " Not an object! Overwriting bailout kind"); + bailoutKind_ = BailoutKind::ThrowCheckIsObject; + } + } + + return true; +} + +bool BaselineStackBuilder::buildFinallyException() { + MOZ_ASSERT(resumingInFinallyBlock()); + + if (!writeValue(excInfo_->finallyException(), "Exception")) { + return false; + } + if (!writeValue(BooleanValue(true), "throwing")) { + return false; + } + + return true; +} + +bool BaselineStackBuilder::prepareForNextFrame( + HandleValueVector savedCallerArgs) { + const uint32_t frameSize = framePushed(); + + // Write out descriptor and return address for the baseline frame. + // The icEntry in question MUST have an inlinable fallback stub. + if (!finishOuterFrame()) { + return false; + } + + return buildStubFrame(frameSize, savedCallerArgs); +} + +bool BaselineStackBuilder::finishOuterFrame() { + // . . + // | Descr(BLJS) | + // +---------------+ + // | ReturnAddr | + // +===============+ + + const BaselineInterpreter& baselineInterp = + cx_->runtime()->jitRuntime()->baselineInterpreter(); + + blFrame()->setInterpreterFields(script_, pc_); + + // Write out descriptor of BaselineJS frame. + size_t baselineFrameDescr = MakeFrameDescriptor(FrameType::BaselineJS); + if (!writeWord(baselineFrameDescr, "Descriptor")) { + return false; + } + + uint8_t* retAddr = baselineInterp.retAddrForIC(op_); + return writePtr(retAddr, "ReturnAddr"); +} + +bool BaselineStackBuilder::buildStubFrame(uint32_t frameSize, + HandleValueVector savedCallerArgs) { + // Build baseline stub frame: + // +===============+ + // | FramePtr | + // +---------------+ + // | StubPtr | + // +---------------+ + // | Padding? | + // +---------------+ + // | ArgA | + // +---------------+ + // | ... | + // +---------------+ + // | Arg0 | + // +---------------+ + // | ThisV | + // +---------------+ + // | CalleeToken | + // +---------------+ + // | Descr(BLStub) | + // +---------------+ + // | ReturnAddr | + // +===============+ + + JitSpew(JitSpew_BaselineBailouts, " [BASELINE-STUB FRAME]"); + + // Write previous frame pointer (saved earlier). + if (!writePtr(prevFramePtr(), "PrevFramePtr")) { + return false; + } + prevFramePtr_ = virtualPointerAtStackOffset(0); + + // Write stub pointer. + uint32_t pcOff = script_->pcToOffset(pc_); + JitScript* jitScript = script_->jitScript(); + const ICEntry& icEntry = jitScript->icEntryFromPCOffset(pcOff); + ICFallbackStub* fallback = jitScript->fallbackStubForICEntry(&icEntry); + if (!writePtr(fallback, "StubPtr")) { + return false; + } + + // Write out the arguments, copied from the baseline frame. The order + // of the arguments is reversed relative to the baseline frame's stack + // values. + MOZ_ASSERT(IsIonInlinableOp(op_)); + bool pushedNewTarget = IsConstructPC(pc_); + unsigned actualArgc; + Value callee; + if (needToSaveCallerArgs()) { + // For accessors, the arguments are not on the stack anymore, + // but they are copied in a vector and are written here. + callee = savedCallerArgs[0]; + actualArgc = IsSetPropOp(op_) ? 1 : 0; + + // Align the stack based on the number of arguments. + size_t afterFrameSize = + (actualArgc + 1) * sizeof(Value) + JitFrameLayout::Size(); + if (!maybeWritePadding(JitStackAlignment, afterFrameSize, "Padding")) { + return false; + } + + // Push arguments. + MOZ_ASSERT(actualArgc + 2 <= exprStackSlots()); + MOZ_ASSERT(savedCallerArgs.length() == actualArgc + 2); + for (unsigned i = 0; i < actualArgc + 1; i++) { + size_t arg = savedCallerArgs.length() - (i + 1); + if (!writeValue(savedCallerArgs[arg], "ArgVal")) { + return false; + } + } + } else if (resumeMode() == ResumeMode::InlinedFunCall && GET_ARGC(pc_) == 0) { + // When calling FunCall with 0 arguments, we push |undefined| + // for this. See BaselineCacheIRCompiler::pushFunCallArguments. + MOZ_ASSERT(!pushedNewTarget); + actualArgc = 0; + // Align the stack based on pushing |this| and 0 arguments. + size_t afterFrameSize = sizeof(Value) + JitFrameLayout::Size(); + if (!maybeWritePadding(JitStackAlignment, afterFrameSize, "Padding")) { + return false; + } + // Push an undefined value for |this|. + if (!writeValue(UndefinedValue(), "ThisValue")) { + return false; + } + size_t calleeSlot = blFrame()->numValueSlots(frameSize) - 1; + callee = *blFrame()->valueSlot(calleeSlot); + + } else { + MOZ_ASSERT(resumeMode() == ResumeMode::InlinedStandardCall || + resumeMode() == ResumeMode::InlinedFunCall); + actualArgc = GET_ARGC(pc_); + if (resumeMode() == ResumeMode::InlinedFunCall) { + // See BaselineCacheIRCompiler::pushFunCallArguments. + MOZ_ASSERT(actualArgc > 0); + actualArgc--; + } + + // In addition to the formal arguments, we must also push |this|. + // When calling a constructor, we must also push |newTarget|. + uint32_t numArguments = actualArgc + 1 + pushedNewTarget; + + // Align the stack based on the number of arguments. + size_t afterFrameSize = + numArguments * sizeof(Value) + JitFrameLayout::Size(); + if (!maybeWritePadding(JitStackAlignment, afterFrameSize, "Padding")) { + return false; + } + + // Copy the arguments and |this| from the BaselineFrame, in reverse order. + size_t valueSlot = blFrame()->numValueSlots(frameSize) - 1; + size_t calleeSlot = valueSlot - numArguments; + + for (size_t i = valueSlot; i > calleeSlot; i--) { + Value v = *blFrame()->valueSlot(i); + if (!writeValue(v, "ArgVal")) { + return false; + } + } + + callee = *blFrame()->valueSlot(calleeSlot); + } + + // In case these arguments need to be copied on the stack again for a + // rectifier frame, save the framePushed values here for later use. + size_t endOfBaselineStubArgs = framePushed(); + + // Push callee token (must be a JS Function) + JitSpew(JitSpew_BaselineBailouts, " Callee = %016" PRIx64, + callee.asRawBits()); + + JSFunction* calleeFun = &callee.toObject().as(); + if (!writePtr(CalleeToToken(calleeFun, pushedNewTarget), "CalleeToken")) { + return false; + } + const ICEntry& icScriptEntry = icScript_->icEntryFromPCOffset(pcOff); + ICFallbackStub* icScriptFallback = + icScript_->fallbackStubForICEntry(&icScriptEntry); + setNextCallee(calleeFun, icScriptFallback->trialInliningState()); + + // Push BaselineStub frame descriptor + size_t baselineStubFrameDescr = + MakeFrameDescriptorForJitCall(FrameType::BaselineStub, actualArgc); + if (!writeWord(baselineStubFrameDescr, "Descriptor")) { + return false; + } + + // Push return address into ICCall_Scripted stub, immediately after the call. + void* baselineCallReturnAddr = getStubReturnAddress(); + MOZ_ASSERT(baselineCallReturnAddr); + if (!writePtr(baselineCallReturnAddr, "ReturnAddr")) { + return false; + } + + // The stack must be aligned after the callee pushes the frame pointer. + MOZ_ASSERT((framePushed() + sizeof(void*)) % JitStackAlignment == 0); + + // Build a rectifier frame if necessary + if (actualArgc < calleeFun->nargs() && + !buildRectifierFrame(actualArgc, endOfBaselineStubArgs)) { + return false; + } + + return true; +} + +bool BaselineStackBuilder::buildRectifierFrame(uint32_t actualArgc, + size_t endOfBaselineStubArgs) { + // Push a reconstructed rectifier frame. + // +===============+ + // | Padding? | + // +---------------+ + // | UndefinedU | + // +---------------+ + // | ... | + // +---------------+ + // | Undefined0 | + // +---------------+ + // | ArgA | + // +---------------+ + // | ... | + // +---------------+ + // | Arg0 | + // +---------------+ + // | ThisV | + // +---------------+ + // | CalleeToken | + // +---------------+ + // | Descr(Rect) | + // +---------------+ + // | ReturnAddr | + // +===============+ + + JitSpew(JitSpew_BaselineBailouts, " [RECTIFIER FRAME]"); + bool pushedNewTarget = IsConstructPC(pc_); + + if (!writePtr(prevFramePtr(), "PrevFramePtr")) { + return false; + } + prevFramePtr_ = virtualPointerAtStackOffset(0); + + // Align the stack based on the number of arguments. + size_t afterFrameSize = + (nextCallee()->nargs() + 1 + pushedNewTarget) * sizeof(Value) + + RectifierFrameLayout::Size(); + if (!maybeWritePadding(JitStackAlignment, afterFrameSize, "Padding")) { + return false; + } + + // Copy new.target, if necessary. + if (pushedNewTarget) { + size_t newTargetOffset = (framePushed() - endOfBaselineStubArgs) + + (actualArgc + 1) * sizeof(Value); + Value newTargetValue = *valuePointerAtStackOffset(newTargetOffset); + if (!writeValue(newTargetValue, "CopiedNewTarget")) { + return false; + } + } + + // Push undefined for missing arguments. + for (unsigned i = 0; i < (nextCallee()->nargs() - actualArgc); i++) { + if (!writeValue(UndefinedValue(), "FillerVal")) { + return false; + } + } + + // Copy arguments + thisv from BaselineStub frame. + if (!subtract((actualArgc + 1) * sizeof(Value), "CopiedArgs")) { + return false; + } + BufferPointer stubArgsEnd = + pointerAtStackOffset(framePushed() - endOfBaselineStubArgs); + JitSpew(JitSpew_BaselineBailouts, " MemCpy from %p", stubArgsEnd.get()); + memcpy(pointerAtStackOffset(0).get(), stubArgsEnd.get(), + (actualArgc + 1) * sizeof(Value)); + + // Push calleeToken again. + if (!writePtr(CalleeToToken(nextCallee(), pushedNewTarget), "CalleeToken")) { + return false; + } + + // Push rectifier frame descriptor + size_t rectifierFrameDescr = + MakeFrameDescriptorForJitCall(FrameType::Rectifier, actualArgc); + if (!writeWord(rectifierFrameDescr, "Descriptor")) { + return false; + } + + // Push return address into the ArgumentsRectifier code, immediately after the + // ioncode call. + void* rectReturnAddr = + cx_->runtime()->jitRuntime()->getArgumentsRectifierReturnAddr().value; + MOZ_ASSERT(rectReturnAddr); + if (!writePtr(rectReturnAddr, "ReturnAddr")) { + return false; + } + + // The stack must be aligned after the callee pushes the frame pointer. + MOZ_ASSERT((framePushed() + sizeof(void*)) % JitStackAlignment == 0); + + return true; +} + +bool BaselineStackBuilder::finishLastFrame() { + const BaselineInterpreter& baselineInterp = + cx_->runtime()->jitRuntime()->baselineInterpreter(); + + setResumeFramePtr(prevFramePtr()); + + // Compute the native address (within the Baseline Interpreter) that we will + // resume at and initialize the frame's interpreter fields. + uint8_t* resumeAddr; + if (isPrologueBailout()) { + JitSpew(JitSpew_BaselineBailouts, " Resuming into prologue."); + MOZ_ASSERT(pc_ == script_->code()); + blFrame()->setInterpreterFieldsForPrologue(script_); + resumeAddr = baselineInterp.bailoutPrologueEntryAddr(); + } else if (propagatingIonExceptionForDebugMode()) { + // When propagating an exception for debug mode, set the + // resume pc to the throwing pc, so that Debugger hooks report + // the correct pc offset of the throwing op instead of its + // successor. + jsbytecode* throwPC = script_->offsetToPC(iter_.pcOffset()); + blFrame()->setInterpreterFields(script_, throwPC); + resumeAddr = baselineInterp.interpretOpAddr().value; + } else { + jsbytecode* resumePC = getResumePC(); + blFrame()->setInterpreterFields(script_, resumePC); + resumeAddr = baselineInterp.interpretOpAddr().value; + } + setResumeAddr(resumeAddr); + JitSpew(JitSpew_BaselineBailouts, " Set resumeAddr=%p", resumeAddr); + + if (cx_->runtime()->geckoProfiler().enabled()) { + // Register bailout with profiler. + const char* filename = script_->filename(); + if (filename == nullptr) { + filename = ""; + } + unsigned len = strlen(filename) + 200; + UniqueChars buf(js_pod_malloc(len)); + if (buf == nullptr) { + ReportOutOfMemory(cx_); + return false; + } + snprintf(buf.get(), len, "%s %s %s on line %u of %s:%u", + BailoutKindString(bailoutKind()), resumeAfter() ? "after" : "at", + CodeName(op_), PCToLineNumber(script_, pc_), filename, + script_->lineno()); + cx_->runtime()->geckoProfiler().markEvent("Bailout", buf.get()); + } + + return true; +} + +#ifdef DEBUG +// The |envChain| slot must not be optimized out if the currently +// active scope requires any EnvironmentObjects beyond what is +// available at body scope. This checks that scope chain does not +// require any such EnvironmentObjects. +// See also: |CompileInfo::isObservableFrameSlot| +bool BaselineStackBuilder::envChainSlotCanBeOptimized() { + jsbytecode* pc = script_->offsetToPC(iter_.pcOffset()); + Scope* scopeIter = script_->innermostScope(pc); + while (scopeIter != script_->bodyScope()) { + if (!scopeIter || scopeIter->hasEnvironment()) { + return false; + } + scopeIter = scopeIter->enclosing(); + } + return true; +} + +bool jit::AssertBailoutStackDepth(JSContext* cx, JSScript* script, + jsbytecode* pc, ResumeMode mode, + uint32_t exprStackSlots) { + if (IsResumeAfter(mode)) { + pc = GetNextPc(pc); + } + + uint32_t expectedDepth; + bool reachablePC; + if (!ReconstructStackDepth(cx, script, pc, &expectedDepth, &reachablePC)) { + return false; + } + if (!reachablePC) { + return true; + } + + JSOp op = JSOp(*pc); + + if (mode == ResumeMode::InlinedFunCall) { + // For inlined fun.call(this, ...); the reconstructed stack depth will + // include the |this|, but the exprStackSlots won't. + // Exception: if there are no arguments, the depths do match. + MOZ_ASSERT(IsInvokeOp(op)); + if (GET_ARGC(pc) > 0) { + MOZ_ASSERT(expectedDepth == exprStackSlots + 1); + } else { + MOZ_ASSERT(expectedDepth == exprStackSlots); + } + return true; + } + + if (mode == ResumeMode::InlinedAccessor) { + // Accessors coming out of ion are inlined via a complete lie perpetrated by + // the compiler internally. Ion just rearranges the stack, and pretends that + // it looked like a call all along. + // This means that the depth is actually one *more* than expected by the + // interpreter, as there is now a JSFunction, |this| and [arg], rather than + // the expected |this| and [arg]. + // If the inlined accessor is a GetElem operation, the numbers do match, but + // that's just because GetElem expects one more item on the stack. Note that + // none of that was pushed, but it's still reflected in exprStackSlots. + MOZ_ASSERT(IsIonInlinableGetterOrSetterOp(op)); + if (IsGetElemOp(op)) { + MOZ_ASSERT(exprStackSlots == expectedDepth); + } else { + MOZ_ASSERT(exprStackSlots == expectedDepth + 1); + } + return true; + } + + // In all other cases, the depth must match. + MOZ_ASSERT(exprStackSlots == expectedDepth); + return true; +} + +bool BaselineStackBuilder::validateFrame() { + const uint32_t frameSize = framePushed(); + blFrame()->setDebugFrameSize(frameSize); + JitSpew(JitSpew_BaselineBailouts, " FrameSize=%u", frameSize); + + // debugNumValueSlots() is based on the frame size, do some sanity checks. + MOZ_ASSERT(blFrame()->debugNumValueSlots() >= script_->nfixed()); + MOZ_ASSERT(blFrame()->debugNumValueSlots() <= script_->nslots()); + + uint32_t expectedSlots = exprStackSlots(); + if (resumingInFinallyBlock()) { + // If we are resuming in a finally block, we push two extra values on the + // stack (the exception, and |throwing|), so the depth at the resume PC + // should be the depth at the fault PC plus two. + expectedSlots += 2; + } + return AssertBailoutStackDepth(cx_, script_, pc_, resumeMode(), + expectedSlots); +} +#endif + +void* BaselineStackBuilder::getStubReturnAddress() { + const BaselineICFallbackCode& code = + cx_->runtime()->jitRuntime()->baselineICFallbackCode(); + + if (IsGetPropOp(op_)) { + return code.bailoutReturnAddr(BailoutReturnKind::GetProp); + } + if (IsSetPropOp(op_)) { + return code.bailoutReturnAddr(BailoutReturnKind::SetProp); + } + if (IsGetElemOp(op_)) { + return code.bailoutReturnAddr(BailoutReturnKind::GetElem); + } + + // This should be a call op of some kind, now. + MOZ_ASSERT(IsInvokeOp(op_) && !IsSpreadOp(op_)); + if (IsConstructOp(op_)) { + return code.bailoutReturnAddr(BailoutReturnKind::New); + } + return code.bailoutReturnAddr(BailoutReturnKind::Call); +} + +static inline jsbytecode* GetNextNonLoopHeadPc(jsbytecode* pc) { + JSOp op = JSOp(*pc); + switch (op) { + case JSOp::Goto: + return pc + GET_JUMP_OFFSET(pc); + + case JSOp::LoopHead: + case JSOp::Nop: + return GetNextPc(pc); + + default: + return pc; + } +} + +// Returns the pc to resume execution at in Baseline after a bailout. +jsbytecode* BaselineStackBuilder::getResumePC() { + if (resumeAfter()) { + return GetNextPc(pc_); + } + + // If we are resuming at a LoopHead op, resume at the next op to avoid + // a bailout -> enter Ion -> bailout loop with --ion-eager. + // + // Cycles can cause the loop below to not terminate. Empty loops are one + // such example: + // + // L: loophead + // goto L + // + // We do cycle detection below with the "tortoise and the hare" algorithm. + jsbytecode* slowerPc = pc_; + jsbytecode* fasterPc = pc_; + while (true) { + // Advance fasterPc twice as fast as slowerPc. + slowerPc = GetNextNonLoopHeadPc(slowerPc); + fasterPc = GetNextNonLoopHeadPc(fasterPc); + fasterPc = GetNextNonLoopHeadPc(fasterPc); + + // Break on cycles or at the end of goto sequences. + if (fasterPc == slowerPc) { + break; + } + } + + return slowerPc; +} + +bool BaselineStackBuilder::isPrologueBailout() { + // If we are propagating an exception for debug mode, we will not resume + // into baseline code, but instead into HandleExceptionBaseline (i.e., + // never before the prologue). + return iter_.pcOffset() == 0 && !iter_.resumeAfter() && + !propagatingIonExceptionForDebugMode(); +} + +// Build a baseline stack frame. +bool BaselineStackBuilder::buildOneFrame() { + // Build a baseline frame: + // +===============+ + // | PrevFramePtr | <-- initFrame() + // +---------------+ + // | Baseline | <-- buildBaselineFrame() + // | Frame | + // +---------------+ + // | Fixed0 | <-- buildFixedSlots() + // +---------------+ + // | ... | + // +---------------+ + // | FixedF | + // +---------------+ + // | Stack0 | <-- buildExpressionStack() -or- fixupCallerArgs() + // +---------------+ + // | ... | + // +---------------+ If we are building the frame in which we will + // | StackS | <-- resume, we stop here. + // +---------------+ finishLastFrame() sets up the interpreter fields. + // . . + // . . + // . . <-- If there are additional frames inlined into this + // | Descr(BLJS) | one, we finish this frame. We generate a stub + // +---------------+ frame (and maybe also a rectifier frame) between + // | ReturnAddr | this frame and the inlined frame. + // +===============+ See: prepareForNextFrame() + + if (!initFrame()) { + return false; + } + + if (!buildBaselineFrame()) { + return false; + } + + if (fun_ && !buildArguments()) { + return false; + } + + if (!buildFixedSlots()) { + return false; + } + + bool fixedUp = false; + RootedValueVector savedCallerArgs(cx_); + if (iter_.moreFrames() && !fixUpCallerArgs(&savedCallerArgs, &fixedUp)) { + return false; + } + + if (!fixedUp) { + if (!buildExpressionStack()) { + return false; + } + if (resumingInFinallyBlock() && !buildFinallyException()) { + return false; + } + } + +#ifdef DEBUG + if (!validateFrame()) { + return false; + } +#endif + +#ifdef JS_JITSPEW + const uint32_t pcOff = script_->pcToOffset(pc()); + JitSpew(JitSpew_BaselineBailouts, + " Resuming %s pc offset %d (op %s) (line %u) of %s:%u:%u", + resumeAfter() ? "after" : "at", (int)pcOff, CodeName(op_), + PCToLineNumber(script_, pc()), script_->filename(), script_->lineno(), + script_->column()); + JitSpew(JitSpew_BaselineBailouts, " Bailout kind: %s", + BailoutKindString(bailoutKind())); +#endif + + // If this was the last inline frame, or we are bailing out to a catch or + // finally block in this frame, then unpacking is almost done. + if (done()) { + return finishLastFrame(); + } + + // Otherwise, this is an outer frame for an inlined call or + // accessor. We will be building an inner frame. Before that, + // we must create a stub frame, and potentially a rectifier frame. + return prepareForNextFrame(savedCallerArgs); +} + +bool jit::BailoutIonToBaseline(JSContext* cx, JitActivation* activation, + const JSJitFrameIter& iter, + BaselineBailoutInfo** bailoutInfo, + const ExceptionBailoutInfo* excInfo, + BailoutReason reason) { + MOZ_ASSERT(bailoutInfo != nullptr); + MOZ_ASSERT(*bailoutInfo == nullptr); + MOZ_ASSERT(iter.isBailoutJS()); + + // Caller should have saved the exception while we perform the bailout. + MOZ_ASSERT(!cx->isExceptionPending()); + + // Ion bailout can fail due to overrecursion and OOM. In such cases we + // cannot honor any further Debugger hooks on the frame, and need to + // ensure that its Debugger.Frame entry is cleaned up. + auto guardRemoveRematerializedFramesFromDebugger = + mozilla::MakeScopeExit([&] { + activation->removeRematerializedFramesFromDebugger(cx, iter.fp()); + }); + + // Always remove the RInstructionResults from the JitActivation, even in + // case of failures as the stack frame is going away after the bailout. + auto removeIonFrameRecovery = mozilla::MakeScopeExit( + [&] { activation->removeIonFrameRecovery(iter.jsFrame()); }); + + // The caller of the top frame must be one of the following: + // IonJS - Ion calling into Ion. + // BaselineStub - Baseline calling into Ion. + // Entry / WasmToJSJit - Interpreter or other (wasm) calling into Ion. + // Rectifier - Arguments rectifier calling into Ion. + // BaselineJS - Resume'd Baseline, then likely OSR'd into Ion. + MOZ_ASSERT(iter.isBailoutJS()); +#if defined(DEBUG) || defined(JS_JITSPEW) + FrameType prevFrameType = iter.prevType(); + MOZ_ASSERT(JSJitFrameIter::isEntry(prevFrameType) || + prevFrameType == FrameType::IonJS || + prevFrameType == FrameType::BaselineStub || + prevFrameType == FrameType::Rectifier || + prevFrameType == FrameType::IonICCall || + prevFrameType == FrameType::BaselineJS || + prevFrameType == FrameType::BaselineInterpreterEntry); +#endif + + // All incoming frames are going to look like this: + // + // +---------------+ + // | ... | + // +---------------+ + // | Args | + // | ... | + // +---------------+ + // | ThisV | + // +---------------+ + // | ActualArgC | + // +---------------+ + // | CalleeToken | + // +---------------+ + // | Descriptor | + // +---------------+ + // | ReturnAddr | + // +---------------+ + // | ||||| | <---- Overwrite starting here. + // | ||||| | + // | ||||| | + // +---------------+ + + JitSpew(JitSpew_BaselineBailouts, + "Bailing to baseline %s:%u:%u (IonScript=%p) (FrameType=%d)", + iter.script()->filename(), iter.script()->lineno(), + iter.script()->column(), (void*)iter.ionScript(), (int)prevFrameType); + + if (excInfo) { + if (excInfo->catchingException()) { + JitSpew(JitSpew_BaselineBailouts, "Resuming in catch or finally block"); + } + if (excInfo->propagatingIonExceptionForDebugMode()) { + JitSpew(JitSpew_BaselineBailouts, "Resuming in-place for debug mode"); + } + } + + JitSpew(JitSpew_BaselineBailouts, + " Reading from snapshot offset %u size %zu", iter.snapshotOffset(), + iter.ionScript()->snapshotsListSize()); + + iter.script()->updateJitCodeRaw(cx->runtime()); + + // Under a bailout, there is no need to invalidate the frame after + // evaluating the recover instruction, as the invalidation is only needed in + // cases where the frame is introspected ahead of the bailout. + MaybeReadFallback recoverBailout(cx, activation, &iter, + MaybeReadFallback::Fallback_DoNothing); + + // Ensure that all value locations are readable from the SnapshotIterator. + // Get the RInstructionResults from the JitActivation if the frame got + // recovered ahead of the bailout. + SnapshotIterator snapIter(iter, activation->bailoutData()->machineState()); + if (!snapIter.initInstructionResults(recoverBailout)) { + return false; + } + +#ifdef TRACK_SNAPSHOTS + snapIter.spewBailingFrom(); +#endif + + BaselineStackBuilder builder(cx, iter, snapIter, excInfo, reason); + if (!builder.init()) { + return false; + } + + JitSpew(JitSpew_BaselineBailouts, " Incoming frame ptr = %p", + builder.startFrame()); + if (iter.maybeCallee()) { + JitSpew(JitSpew_BaselineBailouts, " Callee function (%s:%u:%u)", + iter.script()->filename(), iter.script()->lineno(), + iter.script()->column()); + } else { + JitSpew(JitSpew_BaselineBailouts, " No callee!"); + } + + if (iter.isConstructing()) { + JitSpew(JitSpew_BaselineBailouts, " Constructing!"); + } else { + JitSpew(JitSpew_BaselineBailouts, " Not constructing!"); + } + + JitSpew(JitSpew_BaselineBailouts, " Restoring frames:"); + + while (true) { + // Skip recover instructions as they are already recovered by + // |initInstructionResults|. + snapIter.settleOnFrame(); + + JitSpew(JitSpew_BaselineBailouts, " FrameNo %zu", builder.frameNo()); + + if (!builder.buildOneFrame()) { + MOZ_ASSERT(cx->isExceptionPending()); + return false; + } + + if (builder.done()) { + break; + } + + builder.nextFrame(); + } + JitSpew(JitSpew_BaselineBailouts, " Done restoring frames"); + + BailoutKind bailoutKind = builder.bailoutKind(); + + if (!builder.outermostFrameFormals().empty()) { + // Set the first frame's formals, see the comment in InitFromBailout. + Value* argv = builder.startFrame()->actualArgs(); + mozilla::PodCopy(argv, builder.outermostFrameFormals().begin(), + builder.outermostFrameFormals().length()); + } + + // Do stack check. + bool overRecursed = false; + BaselineBailoutInfo* info = builder.info(); + size_t numBytesToPush = info->copyStackTop - info->copyStackBottom; + MOZ_ASSERT((numBytesToPush % sizeof(uintptr_t)) == 0); + uint8_t* newsp = info->incomingStack - numBytesToPush; +#ifdef JS_SIMULATOR + if (Simulator::Current()->overRecursed(uintptr_t(newsp))) { + overRecursed = true; + } +#else + AutoCheckRecursionLimit recursion(cx); + if (!recursion.checkWithStackPointerDontReport(cx, newsp)) { + overRecursed = true; + } +#endif + if (overRecursed) { + JitSpew(JitSpew_BaselineBailouts, " Overrecursion check failed!"); + ReportOverRecursed(cx); + return false; + } + + // Take the reconstructed baseline stack so it doesn't get freed when builder + // destructs. + info = builder.takeBuffer(); + info->numFrames = builder.frameNo() + 1; + info->bailoutKind.emplace(bailoutKind); + *bailoutInfo = info; + guardRemoveRematerializedFramesFromDebugger.release(); + return true; +} + +static void InvalidateAfterBailout(JSContext* cx, HandleScript outerScript, + const char* reason) { + // In some cases, the computation of recover instruction can invalidate the + // Ion script before we reach the end of the bailout. Thus, if the outer + // script no longer have any Ion script attached, then we just skip the + // invalidation. + // + // For example, such case can happen if the template object for an unboxed + // objects no longer match the content of its properties (see Bug 1174547) + if (!outerScript->hasIonScript()) { + JitSpew(JitSpew_BaselineBailouts, "Ion script is already invalidated"); + return; + } + + MOZ_ASSERT(!outerScript->ionScript()->invalidated()); + + JitSpew(JitSpew_BaselineBailouts, "Invalidating due to %s", reason); + Invalidate(cx, outerScript); +} + +static void HandleLexicalCheckFailure(JSContext* cx, HandleScript outerScript, + HandleScript innerScript) { + JitSpew(JitSpew_IonBailouts, + "Lexical check failure %s:%u:%u, inlined into %s:%u:%u", + innerScript->filename(), innerScript->lineno(), innerScript->column(), + outerScript->filename(), outerScript->lineno(), + outerScript->column()); + + if (!innerScript->failedLexicalCheck()) { + innerScript->setFailedLexicalCheck(); + } + + InvalidateAfterBailout(cx, outerScript, "lexical check failure"); + if (innerScript->hasIonScript()) { + Invalidate(cx, innerScript); + } +} + +static bool CopyFromRematerializedFrame(JSContext* cx, JitActivation* act, + uint8_t* fp, size_t inlineDepth, + BaselineFrame* frame) { + RematerializedFrame* rematFrame = + act->lookupRematerializedFrame(fp, inlineDepth); + + // We might not have rematerialized a frame if the user never requested a + // Debugger.Frame for it. + if (!rematFrame) { + return true; + } + + MOZ_ASSERT(rematFrame->script() == frame->script()); + MOZ_ASSERT(rematFrame->numActualArgs() == frame->numActualArgs()); + + frame->setEnvironmentChain(rematFrame->environmentChain()); + + if (frame->isFunctionFrame()) { + frame->thisArgument() = rematFrame->thisArgument(); + } + + for (unsigned i = 0; i < frame->numActualArgs(); i++) { + frame->argv()[i] = rematFrame->argv()[i]; + } + + for (size_t i = 0; i < frame->script()->nfixed(); i++) { + *frame->valueSlot(i) = rematFrame->locals()[i]; + } + + if (frame->script()->noScriptRval()) { + frame->setReturnValue(UndefinedValue()); + } else { + frame->setReturnValue(rematFrame->returnValue()); + } + + // Don't copy over the hasCachedSavedFrame bit. The new BaselineFrame we're + // building has a different AbstractFramePtr, so it won't be found in the + // LiveSavedFrameCache if we look there. + + JitSpew(JitSpew_BaselineBailouts, + " Copied from rematerialized frame at (%p,%zu)", fp, inlineDepth); + + // Propagate the debuggee frame flag. For the case where the Debugger did + // not rematerialize an Ion frame, the baseline frame has its debuggee + // flag set iff its script is considered a debuggee. See the debuggee case + // in InitFromBailout. + if (rematFrame->isDebuggee()) { + frame->setIsDebuggee(); + return DebugAPI::handleIonBailout(cx, rematFrame, frame); + } + + return true; +} + +enum class BailoutAction { + InvalidateImmediately, + InvalidateIfFrequent, + DisableIfFrequent, + NoAction +}; + +bool jit::FinishBailoutToBaseline(BaselineBailoutInfo* bailoutInfoArg) { + JitSpew(JitSpew_BaselineBailouts, " Done restoring frames"); + + // Use UniquePtr to free the bailoutInfo before we return. + UniquePtr bailoutInfo(bailoutInfoArg); + bailoutInfoArg = nullptr; + + MOZ_DIAGNOSTIC_ASSERT(*bailoutInfo->bailoutKind != BailoutKind::Unreachable); + + JSContext* cx = TlsContext.get(); + + // jit::Bailout(), jit::InvalidationBailout(), and jit::HandleException() + // should have reset the counter to zero. + MOZ_ASSERT(!cx->isInUnsafeRegion()); + + BaselineFrame* topFrame = GetTopBaselineFrame(cx); + + // We have to get rid of the rematerialized frame, whether it is + // restored or unwound. + uint8_t* incomingStack = bailoutInfo->incomingStack; + auto guardRemoveRematerializedFramesFromDebugger = + mozilla::MakeScopeExit([&] { + JitActivation* act = cx->activation()->asJit(); + act->removeRematerializedFramesFromDebugger(cx, incomingStack); + }); + + // Ensure the frame has a call object if it needs one. + if (!EnsureHasEnvironmentObjects(cx, topFrame)) { + return false; + } + + // Create arguments objects for bailed out frames, to maintain the invariant + // that script->needsArgsObj() implies frame->hasArgsObj(). + RootedScript innerScript(cx, nullptr); + RootedScript outerScript(cx, nullptr); + + MOZ_ASSERT(cx->currentlyRunningInJit()); + JSJitFrameIter iter(cx->activation()->asJit()); + uint8_t* outerFp = nullptr; + + // Iter currently points at the exit frame. Get the previous frame + // (which must be a baseline frame), and set it as the last profiling + // frame. + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled( + cx->runtime())) { + MOZ_ASSERT(iter.prevType() == FrameType::BaselineJS); + JitFrameLayout* fp = reinterpret_cast(iter.prevFp()); + cx->jitActivation->setLastProfilingFrame(fp); + } + + uint32_t numFrames = bailoutInfo->numFrames; + MOZ_ASSERT(numFrames > 0); + + uint32_t frameno = 0; + while (frameno < numFrames) { + MOZ_ASSERT(!iter.isIonJS()); + + if (iter.isBaselineJS()) { + BaselineFrame* frame = iter.baselineFrame(); + MOZ_ASSERT(frame->script()->hasBaselineScript()); + + // If the frame doesn't even have a env chain set yet, then it's resuming + // into the the prologue before the env chain is initialized. Any + // necessary args object will also be initialized there. + if (frame->environmentChain() && frame->script()->needsArgsObj()) { + ArgumentsObject* argsObj; + if (frame->hasArgsObj()) { + argsObj = &frame->argsObj(); + } else { + argsObj = ArgumentsObject::createExpected(cx, frame); + if (!argsObj) { + return false; + } + } + + // The arguments is a local binding and needsArgsObj does not + // check if it is clobbered. Ensure that the local binding + // restored during bailout before storing the arguments object + // to the slot. + RootedScript script(cx, frame->script()); + SetFrameArgumentsObject(cx, frame, script, argsObj); + } + + if (frameno == 0) { + innerScript = frame->script(); + } + + if (frameno == numFrames - 1) { + outerScript = frame->script(); + outerFp = iter.fp(); + MOZ_ASSERT(outerFp == incomingStack); + } + + frameno++; + } + + ++iter; + } + + MOZ_ASSERT(innerScript); + MOZ_ASSERT(outerScript); + MOZ_ASSERT(outerFp); + + // If we rematerialized Ion frames due to debug mode toggling, copy their + // values into the baseline frame. We need to do this even when debug mode + // is off, as we should respect the mutations made while debug mode was + // on. + JitActivation* act = cx->activation()->asJit(); + if (act->hasRematerializedFrame(outerFp)) { + JSJitFrameIter iter(act); + size_t inlineDepth = numFrames; + bool ok = true; + while (inlineDepth > 0) { + if (iter.isBaselineJS()) { + // We must attempt to copy all rematerialized frames over, + // even if earlier ones failed, to invoke the proper frame + // cleanup in the Debugger. + if (!CopyFromRematerializedFrame(cx, act, outerFp, --inlineDepth, + iter.baselineFrame())) { + ok = false; + } + } + ++iter; + } + + if (!ok) { + return false; + } + + // After copying from all the rematerialized frames, remove them from + // the table to keep the table up to date. + guardRemoveRematerializedFramesFromDebugger.release(); + act->removeRematerializedFrame(outerFp); + } + + // If we are unwinding for an exception, we need to unwind scopes. + // See |SettleOnTryNote| + if (bailoutInfo->faultPC) { + EnvironmentIter ei(cx, topFrame, bailoutInfo->faultPC); + UnwindEnvironment(cx, ei, bailoutInfo->tryPC); + } + + // Check for interrupts now because we might miss an interrupt check in JIT + // code when resuming in the prologue, after the stack/interrupt check. + if (!cx->isExceptionPending()) { + if (!CheckForInterrupt(cx)) { + return false; + } + } + + BailoutKind bailoutKind = *bailoutInfo->bailoutKind; + JitSpew(JitSpew_BaselineBailouts, + " Restored outerScript=(%s:%u:%u,%u) innerScript=(%s:%u:%u,%u) " + "(bailoutKind=%u)", + outerScript->filename(), outerScript->lineno(), outerScript->column(), + outerScript->getWarmUpCount(), innerScript->filename(), + innerScript->lineno(), innerScript->column(), + innerScript->getWarmUpCount(), (unsigned)bailoutKind); + + BailoutAction action = BailoutAction::InvalidateImmediately; + DebugOnly saveFailedICHash = false; + switch (bailoutKind) { + case BailoutKind::TranspiledCacheIR: + // A transpiled guard failed. If this happens often enough, we will + // invalidate and recompile. + action = BailoutAction::InvalidateIfFrequent; + saveFailedICHash = true; + break; + + case BailoutKind::MonomorphicInlinedStubFolding: + action = BailoutAction::InvalidateIfFrequent; + saveFailedICHash = true; + if (innerScript != outerScript) { + // In the case where this instruction comes from a monomorphic-inlined + // ICScript, we need to ensure that we note the connection between the + // inner script and the outer script, so that we can properly track if + // we add a new case to the folded stub and avoid invalidating the + // outer script. + cx->zone()->jitZone()->noteStubFoldingBailout(innerScript, outerScript); + } + break; + + case BailoutKind::SpeculativePhi: + // A value of an unexpected type flowed into a phi. + MOZ_ASSERT(!outerScript->hadSpeculativePhiBailout()); + outerScript->setHadSpeculativePhiBailout(); + InvalidateAfterBailout(cx, outerScript, "phi specialization failure"); + break; + + case BailoutKind::TypePolicy: + // A conversion inserted by a type policy failed. + // We will invalidate and disable recompilation if this happens too often. + action = BailoutAction::DisableIfFrequent; + break; + + case BailoutKind::LICM: + // LICM may cause spurious bailouts by hoisting unreachable + // guards past branches. To prevent bailout loops, when an + // instruction hoisted by LICM bails out, we update the + // IonScript and resume in baseline. If the guard would have + // been executed anyway, then we will hit the baseline fallback, + // and call noteBaselineFallback. If that does not happen, + // then the next time we reach this point, we will disable LICM + // for this script. + MOZ_ASSERT(!outerScript->hadLICMInvalidation()); + if (outerScript->hasIonScript()) { + switch (outerScript->ionScript()->licmState()) { + case IonScript::LICMState::NeverBailed: + outerScript->ionScript()->setHadLICMBailout(); + action = BailoutAction::NoAction; + break; + case IonScript::LICMState::Bailed: + outerScript->setHadLICMInvalidation(); + InvalidateAfterBailout(cx, outerScript, "LICM failure"); + break; + case IonScript::LICMState::BailedAndHitFallback: + // This bailout is not due to LICM. Treat it like a + // regular TranspiledCacheIR bailout. + action = BailoutAction::InvalidateIfFrequent; + break; + } + } + break; + + case BailoutKind::InstructionReordering: + // An instruction moved up by instruction reordering bailed out. + outerScript->setHadReorderingBailout(); + action = BailoutAction::InvalidateIfFrequent; + break; + + case BailoutKind::HoistBoundsCheck: + // An instruction hoisted or generated by tryHoistBoundsCheck bailed out. + MOZ_ASSERT(!outerScript->failedBoundsCheck()); + outerScript->setFailedBoundsCheck(); + InvalidateAfterBailout(cx, outerScript, "bounds check failure"); + break; + + case BailoutKind::EagerTruncation: + // An eager truncation generated by range analysis bailed out. + // To avoid bailout loops, we set a flag to avoid generating + // eager truncations next time we recompile. + MOZ_ASSERT(!outerScript->hadEagerTruncationBailout()); + outerScript->setHadEagerTruncationBailout(); + InvalidateAfterBailout(cx, outerScript, "eager range analysis failure"); + break; + + case BailoutKind::UnboxFolding: + // An unbox that was hoisted to fold with a load bailed out. + // To avoid bailout loops, we set a flag to avoid folding + // loads with unboxes next time we recompile. + MOZ_ASSERT(!outerScript->hadUnboxFoldingBailout()); + outerScript->setHadUnboxFoldingBailout(); + InvalidateAfterBailout(cx, outerScript, "unbox folding failure"); + break; + + case BailoutKind::TooManyArguments: + // A funapply or spread call had more than JIT_ARGS_LENGTH_MAX arguments. + // We will invalidate and disable recompilation if this happens too often. + action = BailoutAction::DisableIfFrequent; + break; + + case BailoutKind::DuringVMCall: + if (cx->isExceptionPending()) { + // We are bailing out to catch an exception. We will invalidate + // and disable recompilation if this happens too often. + action = BailoutAction::DisableIfFrequent; + } + break; + + case BailoutKind::Finally: + // We are bailing out for a finally block. We will invalidate + // and disable recompilation if this happens too often. + action = BailoutAction::DisableIfFrequent; + break; + + case BailoutKind::Inevitable: + case BailoutKind::Debugger: + // Do nothing. + action = BailoutAction::NoAction; + break; + + case BailoutKind::FirstExecution: + // We reached an instruction that had not been executed yet at + // the time we compiled. If this happens often enough, we will + // invalidate and recompile. + action = BailoutAction::InvalidateIfFrequent; + saveFailedICHash = true; + break; + + case BailoutKind::UninitializedLexical: + HandleLexicalCheckFailure(cx, outerScript, innerScript); + break; + + case BailoutKind::ThrowCheckIsObject: + MOZ_ASSERT(!cx->isExceptionPending()); + return ThrowCheckIsObject(cx, CheckIsObjectKind::IteratorReturn); + + case BailoutKind::IonExceptionDebugMode: + // Return false to resume in HandleException with reconstructed + // baseline frame. + return false; + + case BailoutKind::OnStackInvalidation: + // The script has already been invalidated. There is nothing left to do. + action = BailoutAction::NoAction; + break; + + default: + MOZ_CRASH("Unknown bailout kind!"); + } + +#ifdef DEBUG + if (MOZ_UNLIKELY(cx->runtime()->jitRuntime()->ionBailAfterEnabled())) { + action = BailoutAction::NoAction; + } +#endif + + if (outerScript->hasIonScript()) { + IonScript* ionScript = outerScript->ionScript(); + switch (action) { + case BailoutAction::InvalidateImmediately: + // The IonScript should already have been invalidated. + MOZ_ASSERT(false); + break; + case BailoutAction::InvalidateIfFrequent: + ionScript->incNumFixableBailouts(); + if (ionScript->shouldInvalidate()) { +#ifdef DEBUG + if (saveFailedICHash && !JitOptions.disableBailoutLoopCheck) { + outerScript->jitScript()->setFailedICHash(ionScript->icHash()); + } +#endif + InvalidateAfterBailout(cx, outerScript, "fixable bailouts"); + } + break; + case BailoutAction::DisableIfFrequent: + ionScript->incNumUnfixableBailouts(); + if (ionScript->shouldInvalidateAndDisable()) { + InvalidateAfterBailout(cx, outerScript, "unfixable bailouts"); + outerScript->disableIon(); + } + break; + case BailoutAction::NoAction: + break; + } + } + + return true; +} diff --git a/js/src/jit/BaselineCacheIRCompiler.cpp b/js/src/jit/BaselineCacheIRCompiler.cpp new file mode 100644 index 0000000000..083d0d7c34 --- /dev/null +++ b/js/src/jit/BaselineCacheIRCompiler.cpp @@ -0,0 +1,4083 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/BaselineCacheIRCompiler.h" + +#include "gc/GC.h" +#include "jit/CacheIR.h" +#include "jit/CacheIRCloner.h" +#include "jit/CacheIRWriter.h" +#include "jit/JitFrames.h" +#include "jit/JitRealm.h" +#include "jit/JitRuntime.h" +#include "jit/JitZone.h" +#include "jit/Linker.h" +#include "jit/MoveEmitter.h" +#include "jit/RegExpStubConstants.h" +#include "jit/SharedICHelpers.h" +#include "jit/VMFunctions.h" +#include "js/experimental/JitInfo.h" // JSJitInfo +#include "js/friend/DOMProxy.h" // JS::ExpandoAndGeneration +#include "proxy/DeadObjectProxy.h" +#include "proxy/Proxy.h" +#include "util/Unicode.h" +#include "vm/JSAtom.h" +#include "vm/StaticStrings.h" + +#include "jit/JitScript-inl.h" +#include "jit/MacroAssembler-inl.h" +#include "jit/SharedICHelpers-inl.h" +#include "jit/VMFunctionList-inl.h" +#include "vm/List-inl.h" + +using namespace js; +using namespace js::jit; + +using mozilla::Maybe; + +using JS::ExpandoAndGeneration; + +namespace js { +namespace jit { + +Address CacheRegisterAllocator::addressOf(MacroAssembler& masm, + BaselineFrameSlot slot) const { + uint32_t offset = + stackPushed_ + ICStackValueOffset + slot.slot() * sizeof(JS::Value); + if (JitOptions.enableICFramePointers) { + // The frame pointer is also on the stack. + offset += sizeof(uintptr_t); + } + return Address(masm.getStackPointer(), offset); +} +BaseValueIndex CacheRegisterAllocator::addressOf(MacroAssembler& masm, + Register argcReg, + BaselineFrameSlot slot) const { + uint32_t offset = + stackPushed_ + ICStackValueOffset + slot.slot() * sizeof(JS::Value); + if (JitOptions.enableICFramePointers) { + // The frame pointer is also on the stack. + offset += sizeof(uintptr_t); + } + return BaseValueIndex(masm.getStackPointer(), argcReg, offset); +} + +// BaselineCacheIRCompiler compiles CacheIR to BaselineIC native code. +BaselineCacheIRCompiler::BaselineCacheIRCompiler(JSContext* cx, + TempAllocator& alloc, + const CacheIRWriter& writer, + uint32_t stubDataOffset) + : CacheIRCompiler(cx, alloc, writer, stubDataOffset, Mode::Baseline, + StubFieldPolicy::Address), + makesGCCalls_(false) {} + +// AutoStubFrame methods +AutoStubFrame::AutoStubFrame(BaselineCacheIRCompiler& compiler) + : compiler(compiler) +#ifdef DEBUG + , + framePushedAtEnterStubFrame_(0) +#endif +{ +} +void AutoStubFrame::enter(MacroAssembler& masm, Register scratch, + CallCanGC canGC) { + MOZ_ASSERT(compiler.allocator.stackPushed() == 0); + + if (JitOptions.enableICFramePointers) { + // If we have already pushed the frame pointer, pop it + // before creating the stub frame. + masm.pop(FramePointer); + } + EmitBaselineEnterStubFrame(masm, scratch); + +#ifdef DEBUG + framePushedAtEnterStubFrame_ = masm.framePushed(); +#endif + + MOZ_ASSERT(!compiler.enteredStubFrame_); + compiler.enteredStubFrame_ = true; + if (canGC == CallCanGC::CanGC) { + compiler.makesGCCalls_ = true; + } +} +void AutoStubFrame::leave(MacroAssembler& masm) { + MOZ_ASSERT(compiler.enteredStubFrame_); + compiler.enteredStubFrame_ = false; + +#ifdef DEBUG + masm.setFramePushed(framePushedAtEnterStubFrame_); +#endif + + EmitBaselineLeaveStubFrame(masm); + if (JitOptions.enableICFramePointers) { + // We will pop the frame pointer when we return, + // so we have to push it again now. + masm.push(FramePointer); + } +} + +#ifdef DEBUG +AutoStubFrame::~AutoStubFrame() { MOZ_ASSERT(!compiler.enteredStubFrame_); } +#endif + +} // namespace jit +} // namespace js + +bool BaselineCacheIRCompiler::makesGCCalls() const { return makesGCCalls_; } + +Address BaselineCacheIRCompiler::stubAddress(uint32_t offset) const { + return Address(ICStubReg, stubDataOffset_ + offset); +} + +template +void BaselineCacheIRCompiler::callVM(MacroAssembler& masm) { + VMFunctionId id = VMFunctionToId::id; + callVMInternal(masm, id); +} + +JitCode* BaselineCacheIRCompiler::compile() { + AutoCreatedBy acb(masm, "BaselineCacheIRCompiler::compile"); + +#ifndef JS_USE_LINK_REGISTER + masm.adjustFrame(sizeof(intptr_t)); +#endif +#ifdef JS_CODEGEN_ARM + masm.setSecondScratchReg(BaselineSecondScratchReg); +#endif + if (JitOptions.enableICFramePointers) { + /* [SMDOC] Baseline IC Frame Pointers + * + * In general, ICs don't have frame pointers until just before + * doing a VM call, at which point we retroactively create a stub + * frame. However, for the sake of external profilers, we + * optionally support full-IC frame pointers in baseline ICs, with + * the following approach: + * 1. We push a frame pointer when we enter an IC. + * 2. We pop the frame pointer when we return from an IC, or + * when we jump to the next IC. + * 3. Entering a stub frame for a VM call already pushes a + * frame pointer, so we pop our existing frame pointer + * just before entering a stub frame and push it again + * just after leaving a stub frame. + * Some ops take advantage of the fact that the frame pointer is + * not updated until we enter a stub frame to read values from + * the caller's frame. To support this, we allocate a separate + * baselineFrame register when IC frame pointers are enabled. + */ + masm.push(FramePointer); + masm.moveStackPtrTo(FramePointer); + + MOZ_ASSERT(baselineFrameReg() != FramePointer); + masm.loadPtr(Address(FramePointer, 0), baselineFrameReg()); + } + + // Count stub entries: We count entries rather than successes as it much + // easier to ensure ICStubReg is valid at entry than at exit. + Address enteredCount(ICStubReg, ICCacheIRStub::offsetOfEnteredCount()); + masm.add32(Imm32(1), enteredCount); + + CacheIRReader reader(writer_); + do { + CacheOp op = reader.readOp(); + perfSpewer_.recordInstruction(masm, op); + switch (op) { +#define DEFINE_OP(op, ...) \ + case CacheOp::op: \ + if (!emit##op(reader)) return nullptr; \ + break; + CACHE_IR_OPS(DEFINE_OP) +#undef DEFINE_OP + + default: + MOZ_CRASH("Invalid op"); + } + allocator.nextOp(); + } while (reader.more()); + + MOZ_ASSERT(!enteredStubFrame_); + masm.assumeUnreachable("Should have returned from IC"); + + // Done emitting the main IC code. Now emit the failure paths. + for (size_t i = 0; i < failurePaths.length(); i++) { + if (!emitFailurePath(i)) { + return nullptr; + } + if (JitOptions.enableICFramePointers) { + masm.pop(FramePointer); + } + EmitStubGuardFailure(masm); + } + + Linker linker(masm); + Rooted newStubCode(cx_, linker.newCode(cx_, CodeKind::Baseline)); + if (!newStubCode) { + cx_->recoverFromOutOfMemory(); + return nullptr; + } + + return newStubCode; +} + +bool BaselineCacheIRCompiler::emitGuardShape(ObjOperandId objId, + uint32_t shapeOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + AutoScratchRegister scratch1(allocator, masm); + + bool needSpectreMitigations = objectGuardNeedsSpectreMitigations(objId); + + Maybe maybeScratch2; + if (needSpectreMitigations) { + maybeScratch2.emplace(allocator, masm); + } + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + Address addr(stubAddress(shapeOffset)); + masm.loadPtr(addr, scratch1); + if (needSpectreMitigations) { + masm.branchTestObjShape(Assembler::NotEqual, obj, scratch1, *maybeScratch2, + obj, failure->label()); + } else { + masm.branchTestObjShapeNoSpectreMitigations(Assembler::NotEqual, obj, + scratch1, failure->label()); + } + + return true; +} + +bool BaselineCacheIRCompiler::emitGuardProto(ObjOperandId objId, + uint32_t protoOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + Address addr(stubAddress(protoOffset)); + masm.loadObjProto(obj, scratch); + masm.branchPtr(Assembler::NotEqual, addr, scratch, failure->label()); + return true; +} + +bool BaselineCacheIRCompiler::emitGuardCompartment(ObjOperandId objId, + uint32_t globalOffset, + uint32_t compartmentOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + // Verify that the global wrapper is still valid, as + // it is pre-requisite for doing the compartment check. + Address globalWrapper(stubAddress(globalOffset)); + masm.loadPtr(globalWrapper, scratch); + Address handlerAddr(scratch, ProxyObject::offsetOfHandler()); + masm.branchPtr(Assembler::Equal, handlerAddr, + ImmPtr(&DeadObjectProxy::singleton), failure->label()); + + Address addr(stubAddress(compartmentOffset)); + masm.branchTestObjCompartment(Assembler::NotEqual, obj, addr, scratch, + failure->label()); + return true; +} + +bool BaselineCacheIRCompiler::emitGuardAnyClass(ObjOperandId objId, + uint32_t claspOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + Address testAddr(stubAddress(claspOffset)); + if (objectGuardNeedsSpectreMitigations(objId)) { + masm.branchTestObjClass(Assembler::NotEqual, obj, testAddr, scratch, obj, + failure->label()); + } else { + masm.branchTestObjClassNoSpectreMitigations( + Assembler::NotEqual, obj, testAddr, scratch, failure->label()); + } + + return true; +} + +bool BaselineCacheIRCompiler::emitGuardHasProxyHandler(ObjOperandId objId, + uint32_t handlerOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + Address testAddr(stubAddress(handlerOffset)); + masm.loadPtr(testAddr, scratch); + + Address handlerAddr(obj, ProxyObject::offsetOfHandler()); + masm.branchPtr(Assembler::NotEqual, handlerAddr, scratch, failure->label()); + return true; +} + +bool BaselineCacheIRCompiler::emitGuardSpecificObject(ObjOperandId objId, + uint32_t expectedOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + Address addr(stubAddress(expectedOffset)); + masm.branchPtr(Assembler::NotEqual, addr, obj, failure->label()); + return true; +} + +bool BaselineCacheIRCompiler::emitGuardSpecificFunction( + ObjOperandId objId, uint32_t expectedOffset, uint32_t nargsAndFlagsOffset) { + return emitGuardSpecificObject(objId, expectedOffset); +} + +bool BaselineCacheIRCompiler::emitGuardFunctionScript( + ObjOperandId funId, uint32_t expectedOffset, uint32_t nargsAndFlagsOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + Register fun = allocator.useRegister(masm, funId); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + Address addr(stubAddress(expectedOffset)); + masm.loadPrivate(Address(fun, JSFunction::offsetOfJitInfoOrScript()), + scratch); + masm.branchPtr(Assembler::NotEqual, addr, scratch, failure->label()); + return true; +} + +bool BaselineCacheIRCompiler::emitGuardSpecificAtom(StringOperandId strId, + uint32_t expectedOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register str = allocator.useRegister(masm, strId); + AutoScratchRegister scratch(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + Address atomAddr(stubAddress(expectedOffset)); + + Label done; + masm.branchPtr(Assembler::Equal, atomAddr, str, &done); + + // The pointers are not equal, so if the input string is also an atom it + // must be a different string. + masm.branchTest32(Assembler::NonZero, Address(str, JSString::offsetOfFlags()), + Imm32(JSString::ATOM_BIT), failure->label()); + + // Check the length. + masm.loadPtr(atomAddr, scratch); + masm.loadStringLength(scratch, scratch); + masm.branch32(Assembler::NotEqual, Address(str, JSString::offsetOfLength()), + scratch, failure->label()); + + // We have a non-atomized string with the same length. Call a helper + // function to do the comparison. + LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(), + liveVolatileFloatRegs()); + masm.PushRegsInMask(volatileRegs); + + using Fn = bool (*)(JSString* str1, JSString* str2); + masm.setupUnalignedABICall(scratch); + masm.loadPtr(atomAddr, scratch); + masm.passABIArg(scratch); + masm.passABIArg(str); + masm.callWithABI(); + masm.storeCallPointerResult(scratch); + + LiveRegisterSet ignore; + ignore.add(scratch); + masm.PopRegsInMaskIgnore(volatileRegs, ignore); + masm.branchIfFalseBool(scratch, failure->label()); + + masm.bind(&done); + return true; +} + +bool BaselineCacheIRCompiler::emitGuardSpecificSymbol(SymbolOperandId symId, + uint32_t expectedOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register sym = allocator.useRegister(masm, symId); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + Address addr(stubAddress(expectedOffset)); + masm.branchPtr(Assembler::NotEqual, addr, sym, failure->label()); + return true; +} + +bool BaselineCacheIRCompiler::emitLoadValueResult(uint32_t valOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + AutoOutputRegister output(*this); + masm.loadValue(stubAddress(valOffset), output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitLoadFixedSlotResult(ObjOperandId objId, + uint32_t offsetOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, objId); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + masm.load32(stubAddress(offsetOffset), scratch); + masm.loadValue(BaseIndex(obj, scratch, TimesOne), output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitLoadFixedSlotTypedResult( + ObjOperandId objId, uint32_t offsetOffset, ValueType) { + // The type is only used by Warp. + return emitLoadFixedSlotResult(objId, offsetOffset); +} + +bool BaselineCacheIRCompiler::emitLoadDynamicSlotResult(ObjOperandId objId, + uint32_t offsetOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, objId); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + AutoScratchRegister scratch2(allocator, masm); + + masm.load32(stubAddress(offsetOffset), scratch); + masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch2); + masm.loadValue(BaseIndex(scratch2, scratch, TimesOne), output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitCallScriptedGetterShared( + ValOperandId receiverId, uint32_t getterOffset, bool sameRealm, + uint32_t nargsAndFlagsOffset, Maybe icScriptOffset) { + ValueOperand receiver = allocator.useValueRegister(masm, receiverId); + Address getterAddr(stubAddress(getterOffset)); + + AutoScratchRegister code(allocator, masm); + AutoScratchRegister callee(allocator, masm); + AutoScratchRegister scratch(allocator, masm); + + bool isInlined = icScriptOffset.isSome(); + + // First, retrieve raw jitcode for getter. + masm.loadPtr(getterAddr, callee); + if (isInlined) { + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + masm.loadBaselineJitCodeRaw(callee, code, failure->label()); + } else { + masm.loadJitCodeRaw(callee, code); + } + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + if (!sameRealm) { + masm.switchToObjectRealm(callee, scratch); + } + + // Align the stack such that the JitFrameLayout is aligned on + // JitStackAlignment. + masm.alignJitStackBasedOnNArgs(0, /*countIncludesThis = */ false); + + // Getter is called with 0 arguments, just |receiver| as thisv. + // Note that we use Push, not push, so that callJit will align the stack + // properly on ARM. + masm.Push(receiver); + + if (isInlined) { + // Store icScript in the context. + Address icScriptAddr(stubAddress(*icScriptOffset)); + masm.loadPtr(icScriptAddr, scratch); + masm.storeICScriptInJSContext(scratch); + } + + masm.Push(callee); + masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, /* argc = */ 0); + + // Handle arguments underflow. + Label noUnderflow; + masm.loadFunctionArgCount(callee, callee); + masm.branch32(Assembler::Equal, callee, Imm32(0), &noUnderflow); + + // Call the arguments rectifier. + ArgumentsRectifierKind kind = isInlined + ? ArgumentsRectifierKind::TrialInlining + : ArgumentsRectifierKind::Normal; + TrampolinePtr argumentsRectifier = + cx_->runtime()->jitRuntime()->getArgumentsRectifier(kind); + masm.movePtr(argumentsRectifier, code); + + masm.bind(&noUnderflow); + masm.callJit(code); + + stubFrame.leave(masm); + + if (!sameRealm) { + masm.switchToBaselineFrameRealm(R1.scratchReg()); + } + + return true; +} + +bool BaselineCacheIRCompiler::emitCallScriptedGetterResult( + ValOperandId receiverId, uint32_t getterOffset, bool sameRealm, + uint32_t nargsAndFlagsOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Maybe icScriptOffset = mozilla::Nothing(); + return emitCallScriptedGetterShared(receiverId, getterOffset, sameRealm, + nargsAndFlagsOffset, icScriptOffset); +} + +bool BaselineCacheIRCompiler::emitCallInlinedGetterResult( + ValOperandId receiverId, uint32_t getterOffset, uint32_t icScriptOffset, + bool sameRealm, uint32_t nargsAndFlagsOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + return emitCallScriptedGetterShared(receiverId, getterOffset, sameRealm, + nargsAndFlagsOffset, + mozilla::Some(icScriptOffset)); +} + +bool BaselineCacheIRCompiler::emitCallNativeGetterResult( + ValOperandId receiverId, uint32_t getterOffset, bool sameRealm, + uint32_t nargsAndFlagsOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + ValueOperand receiver = allocator.useValueRegister(masm, receiverId); + Address getterAddr(stubAddress(getterOffset)); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Load the callee in the scratch register. + masm.loadPtr(getterAddr, scratch); + + masm.Push(receiver); + masm.Push(scratch); + + using Fn = + bool (*)(JSContext*, HandleFunction, HandleValue, MutableHandleValue); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitCallDOMGetterResult(ObjOperandId objId, + uint32_t jitInfoOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + Register obj = allocator.useRegister(masm, objId); + Address jitInfoAddr(stubAddress(jitInfoOffset)); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Load the JSJitInfo in the scratch register. + masm.loadPtr(jitInfoAddr, scratch); + + masm.Push(obj); + masm.Push(scratch); + + using Fn = + bool (*)(JSContext*, const JSJitInfo*, HandleObject, MutableHandleValue); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitProxyGetResult(ObjOperandId objId, + uint32_t idOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + Address idAddr(stubAddress(idOffset)); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Load the jsid in the scratch register. + masm.loadPtr(idAddr, scratch); + + masm.Push(scratch); + masm.Push(obj); + + using Fn = bool (*)(JSContext*, HandleObject, HandleId, MutableHandleValue); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitFrameIsConstructingResult() { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register outputScratch = output.valueReg().scratchReg(); + + // Load the CalleeToken. + Address tokenAddr(baselineFrameReg(), JitFrameLayout::offsetOfCalleeToken()); + masm.loadPtr(tokenAddr, outputScratch); + + // The low bit indicates whether this call is constructing, just clear the + // other bits. + static_assert(CalleeToken_Function == 0x0); + static_assert(CalleeToken_FunctionConstructing == 0x1); + masm.andPtr(Imm32(0x1), outputScratch); + + masm.tagValue(JSVAL_TYPE_BOOLEAN, outputScratch, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitLoadConstantStringResult(uint32_t strOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + masm.loadPtr(stubAddress(strOffset), scratch); + masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitCompareStringResult(JSOp op, + StringOperandId lhsId, + StringOperandId rhsId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + AutoOutputRegister output(*this); + + Register left = allocator.useRegister(masm, lhsId); + Register right = allocator.useRegister(masm, rhsId); + + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + allocator.discardStack(masm); + + Label slow, done; + masm.compareStrings(op, left, right, scratch, &slow); + masm.jump(&done); + masm.bind(&slow); + { + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Push the operands in reverse order for JSOp::Le and JSOp::Gt: + // - |left <= right| is implemented as |right >= left|. + // - |left > right| is implemented as |right < left|. + if (op == JSOp::Le || op == JSOp::Gt) { + masm.Push(left); + masm.Push(right); + } else { + masm.Push(right); + masm.Push(left); + } + + using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*); + if (op == JSOp::Eq || op == JSOp::StrictEq) { + callVM>(masm); + } else if (op == JSOp::Ne || op == JSOp::StrictNe) { + callVM>(masm); + } else if (op == JSOp::Lt || op == JSOp::Gt) { + callVM>(masm); + } else { + MOZ_ASSERT(op == JSOp::Le || op == JSOp::Ge); + callVM>(masm); + } + + stubFrame.leave(masm); + masm.storeCallPointerResult(scratch); + } + masm.bind(&done); + masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitSameValueResult(ValOperandId lhsId, + ValOperandId rhsId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegister scratch(allocator, masm); + ValueOperand lhs = allocator.useValueRegister(masm, lhsId); +#ifdef JS_CODEGEN_X86 + // Use the output to avoid running out of registers. + allocator.copyToScratchValueRegister(masm, rhsId, output.valueReg()); + ValueOperand rhs = output.valueReg(); +#else + ValueOperand rhs = allocator.useValueRegister(masm, rhsId); +#endif + + allocator.discardStack(masm); + + Label done; + Label call; + + // Check to see if the values have identical bits. + // This is correct for SameValue because SameValue(NaN,NaN) is true, + // and SameValue(0,-0) is false. + masm.branch64(Assembler::NotEqual, lhs.toRegister64(), rhs.toRegister64(), + &call); + masm.moveValue(BooleanValue(true), output.valueReg()); + masm.jump(&done); + + { + masm.bind(&call); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.pushValue(lhs); + masm.pushValue(rhs); + + using Fn = bool (*)(JSContext*, HandleValue, HandleValue, bool*); + callVM(masm); + + stubFrame.leave(masm); + masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, output.valueReg()); + } + + masm.bind(&done); + return true; +} + +bool BaselineCacheIRCompiler::emitStoreSlotShared(bool isFixed, + ObjOperandId objId, + uint32_t offsetOffset, + ValOperandId rhsId) { + Register obj = allocator.useRegister(masm, objId); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + + AutoScratchRegister scratch1(allocator, masm); + Maybe scratch2; + if (!isFixed) { + scratch2.emplace(allocator, masm); + } + + Address offsetAddr = stubAddress(offsetOffset); + masm.load32(offsetAddr, scratch1); + + if (isFixed) { + BaseIndex slot(obj, scratch1, TimesOne); + EmitPreBarrier(masm, slot, MIRType::Value); + masm.storeValue(val, slot); + } else { + masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch2.ref()); + BaseIndex slot(scratch2.ref(), scratch1, TimesOne); + EmitPreBarrier(masm, slot, MIRType::Value); + masm.storeValue(val, slot); + } + + emitPostBarrierSlot(obj, val, scratch1); + return true; +} + +bool BaselineCacheIRCompiler::emitStoreFixedSlot(ObjOperandId objId, + uint32_t offsetOffset, + ValOperandId rhsId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + return emitStoreSlotShared(true, objId, offsetOffset, rhsId); +} + +bool BaselineCacheIRCompiler::emitStoreDynamicSlot(ObjOperandId objId, + uint32_t offsetOffset, + ValOperandId rhsId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + return emitStoreSlotShared(false, objId, offsetOffset, rhsId); +} + +bool BaselineCacheIRCompiler::emitAddAndStoreSlotShared( + CacheOp op, ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId, + uint32_t newShapeOffset, Maybe numNewSlotsOffset) { + Register obj = allocator.useRegister(masm, objId); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + + Address newShapeAddr = stubAddress(newShapeOffset); + Address offsetAddr = stubAddress(offsetOffset); + + if (op == CacheOp::AllocateAndStoreDynamicSlot) { + // We have to (re)allocate dynamic slots. Do this first, as it's the + // only fallible operation here. Note that growSlotsPure is fallible but + // does not GC. + Address numNewSlotsAddr = stubAddress(*numNewSlotsOffset); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + LiveRegisterSet save(GeneralRegisterSet::Volatile(), + liveVolatileFloatRegs()); + masm.PushRegsInMask(save); + + using Fn = bool (*)(JSContext* cx, NativeObject* obj, uint32_t newCount); + masm.setupUnalignedABICall(scratch1); + masm.loadJSContext(scratch1); + masm.passABIArg(scratch1); + masm.passABIArg(obj); + masm.load32(numNewSlotsAddr, scratch2); + masm.passABIArg(scratch2); + masm.callWithABI(); + masm.storeCallPointerResult(scratch1); + + LiveRegisterSet ignore; + ignore.add(scratch1); + masm.PopRegsInMaskIgnore(save, ignore); + + masm.branchIfFalseBool(scratch1, failure->label()); + } + + // Update the object's shape. + masm.loadPtr(newShapeAddr, scratch1); + masm.storeObjShape(scratch1, obj, + [](MacroAssembler& masm, const Address& addr) { + EmitPreBarrier(masm, addr, MIRType::Shape); + }); + + // Perform the store. No pre-barrier required since this is a new + // initialization. + masm.load32(offsetAddr, scratch1); + if (op == CacheOp::AddAndStoreFixedSlot) { + BaseIndex slot(obj, scratch1, TimesOne); + masm.storeValue(val, slot); + } else { + MOZ_ASSERT(op == CacheOp::AddAndStoreDynamicSlot || + op == CacheOp::AllocateAndStoreDynamicSlot); + masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch2); + BaseIndex slot(scratch2, scratch1, TimesOne); + masm.storeValue(val, slot); + } + + emitPostBarrierSlot(obj, val, scratch1); + return true; +} + +bool BaselineCacheIRCompiler::emitAddAndStoreFixedSlot( + ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId, + uint32_t newShapeOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Maybe numNewSlotsOffset = mozilla::Nothing(); + return emitAddAndStoreSlotShared(CacheOp::AddAndStoreFixedSlot, objId, + offsetOffset, rhsId, newShapeOffset, + numNewSlotsOffset); +} + +bool BaselineCacheIRCompiler::emitAddAndStoreDynamicSlot( + ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId, + uint32_t newShapeOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Maybe numNewSlotsOffset = mozilla::Nothing(); + return emitAddAndStoreSlotShared(CacheOp::AddAndStoreDynamicSlot, objId, + offsetOffset, rhsId, newShapeOffset, + numNewSlotsOffset); +} + +bool BaselineCacheIRCompiler::emitAllocateAndStoreDynamicSlot( + ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId, + uint32_t newShapeOffset, uint32_t numNewSlotsOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + return emitAddAndStoreSlotShared(CacheOp::AllocateAndStoreDynamicSlot, objId, + offsetOffset, rhsId, newShapeOffset, + mozilla::Some(numNewSlotsOffset)); +} + +bool BaselineCacheIRCompiler::emitArrayJoinResult(ObjOperandId objId, + StringOperandId sepId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, objId); + Register sep = allocator.useRegister(masm, sepId); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + allocator.discardStack(masm); + + // Load obj->elements in scratch. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch); + Address lengthAddr(scratch, ObjectElements::offsetOfLength()); + + // If array length is 0, return empty string. + Label finished; + + { + Label arrayNotEmpty; + masm.branch32(Assembler::NotEqual, lengthAddr, Imm32(0), &arrayNotEmpty); + masm.movePtr(ImmGCPtr(cx_->names().empty), scratch); + masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg()); + masm.jump(&finished); + masm.bind(&arrayNotEmpty); + } + + Label vmCall; + + // Otherwise, handle array length 1 case. + masm.branch32(Assembler::NotEqual, lengthAddr, Imm32(1), &vmCall); + + // But only if initializedLength is also 1. + Address initLength(scratch, ObjectElements::offsetOfInitializedLength()); + masm.branch32(Assembler::NotEqual, initLength, Imm32(1), &vmCall); + + // And only if elem0 is a string. + Address elementAddr(scratch, 0); + masm.branchTestString(Assembler::NotEqual, elementAddr, &vmCall); + + // Store the value. + masm.loadValue(elementAddr, output.valueReg()); + masm.jump(&finished); + + // Otherwise call into the VM. + { + masm.bind(&vmCall); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.Push(sep); + masm.Push(obj); + + using Fn = JSString* (*)(JSContext*, HandleObject, HandleString); + callVM(masm); + + stubFrame.leave(masm); + + masm.tagValue(JSVAL_TYPE_STRING, ReturnReg, output.valueReg()); + } + + masm.bind(&finished); + + return true; +} + +bool BaselineCacheIRCompiler::emitPackedArraySliceResult( + uint32_t templateObjectOffset, ObjOperandId arrayId, Int32OperandId beginId, + Int32OperandId endId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output); + AutoScratchRegisterMaybeOutputType scratch2(allocator, masm, output); + + Register array = allocator.useRegister(masm, arrayId); + Register begin = allocator.useRegister(masm, beginId); + Register end = allocator.useRegister(masm, endId); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + masm.branchArrayIsNotPacked(array, scratch1, scratch2, failure->label()); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch1); + + // Don't attempt to pre-allocate the object, instead always use the slow + // path. + ImmPtr result(nullptr); + + masm.Push(result); + masm.Push(end); + masm.Push(begin); + masm.Push(array); + + using Fn = + JSObject* (*)(JSContext*, HandleObject, int32_t, int32_t, HandleObject); + callVM(masm); + + stubFrame.leave(masm); + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitArgumentsSliceResult( + uint32_t templateObjectOffset, ObjOperandId argsId, Int32OperandId beginId, + Int32OperandId endId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + Register args = allocator.useRegister(masm, argsId); + Register begin = allocator.useRegister(masm, beginId); + Register end = allocator.useRegister(masm, endId); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Don't attempt to pre-allocate the object, instead always use the slow path. + ImmPtr result(nullptr); + + masm.Push(result); + masm.Push(end); + masm.Push(begin); + masm.Push(args); + + using Fn = + JSObject* (*)(JSContext*, HandleObject, int32_t, int32_t, HandleObject); + callVM(masm); + + stubFrame.leave(masm); + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitIsArrayResult(ValOperandId inputId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegisterMaybeOutput scratch2(allocator, masm, output); + + ValueOperand val = allocator.useValueRegister(masm, inputId); + + allocator.discardStack(masm); + + Label isNotArray; + // Primitives are never Arrays. + masm.fallibleUnboxObject(val, scratch1, &isNotArray); + + Label isArray; + masm.branchTestObjClass(Assembler::Equal, scratch1, &ArrayObject::class_, + scratch2, scratch1, &isArray); + + // isArray can also return true for Proxy wrapped Arrays. + masm.branchTestObjectIsProxy(false, scratch1, scratch2, &isNotArray); + Label done; + { + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch2); + + masm.Push(scratch1); + + using Fn = bool (*)(JSContext*, HandleObject, bool*); + callVM(masm); + + stubFrame.leave(masm); + + masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, output.valueReg()); + masm.jump(&done); + } + + masm.bind(&isNotArray); + masm.moveValue(BooleanValue(false), output.valueReg()); + masm.jump(&done); + + masm.bind(&isArray); + masm.moveValue(BooleanValue(true), output.valueReg()); + + masm.bind(&done); + return true; +} + +bool BaselineCacheIRCompiler::emitIsTypedArrayResult(ObjOperandId objId, + bool isPossiblyWrapped) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + Register obj = allocator.useRegister(masm, objId); + + allocator.discardStack(masm); + + Label notTypedArray, isProxy, done; + masm.loadObjClassUnsafe(obj, scratch); + masm.branchIfClassIsNotTypedArray(scratch, ¬TypedArray); + masm.moveValue(BooleanValue(true), output.valueReg()); + masm.jump(&done); + + masm.bind(¬TypedArray); + if (isPossiblyWrapped) { + masm.branchTestClassIsProxy(true, scratch, &isProxy); + } + masm.moveValue(BooleanValue(false), output.valueReg()); + + if (isPossiblyWrapped) { + masm.jump(&done); + + masm.bind(&isProxy); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.Push(obj); + + using Fn = bool (*)(JSContext*, JSObject*, bool*); + callVM(masm); + + stubFrame.leave(masm); + + masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, output.valueReg()); + } + + masm.bind(&done); + return true; +} + +bool BaselineCacheIRCompiler::emitLoadStringCharResult(StringOperandId strId, + Int32OperandId indexId, + bool handleOOB) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + AutoOutputRegister output(*this); + Register str = allocator.useRegister(masm, strId); + Register index = allocator.useRegister(masm, indexId); + AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output); + AutoScratchRegisterMaybeOutputType scratch2(allocator, masm, output); + AutoScratchRegister scratch3(allocator, masm); + + // Bounds check, load string char. + Label done; + Label loadFailed; + if (!handleOOB) { + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()), + scratch1, failure->label()); + masm.loadStringChar(str, index, scratch1, scratch2, scratch3, + failure->label()); + + allocator.discardStack(masm); + } else { + // Discard the stack before jumping to |done|. + allocator.discardStack(masm); + + // Return the empty string for out-of-bounds access. + masm.movePtr(ImmGCPtr(cx_->names().empty), scratch2); + + // This CacheIR op is always preceded by |LinearizeForCharAccess|, so we're + // guaranteed to see no nested ropes. + masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()), + scratch1, &done); + masm.loadStringChar(str, index, scratch1, scratch2, scratch3, &loadFailed); + } + + // Load StaticString for this char. For larger code units perform a VM call. + Label vmCall; + masm.boundsCheck32PowerOfTwo(scratch1, StaticStrings::UNIT_STATIC_LIMIT, + &vmCall); + masm.movePtr(ImmPtr(&cx_->staticStrings().unitStaticTable), scratch2); + masm.loadPtr(BaseIndex(scratch2, scratch1, ScalePointer), scratch2); + + masm.jump(&done); + + if (handleOOB) { + masm.bind(&loadFailed); + masm.assumeUnreachable("loadStringChar can't fail for linear strings"); + } + + { + masm.bind(&vmCall); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch2); + + masm.Push(scratch1); + + using Fn = JSLinearString* (*)(JSContext*, int32_t); + callVM(masm); + + stubFrame.leave(masm); + + masm.storeCallPointerResult(scratch2); + } + + masm.bind(&done); + masm.tagValue(JSVAL_TYPE_STRING, scratch2, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitStringFromCodeResult(Int32OperandId codeId, + StringCode stringCode) { + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + Register code = allocator.useRegister(masm, codeId); + + FailurePath* failure = nullptr; + if (stringCode == StringCode::CodePoint) { + if (!addFailurePath(&failure)) { + return false; + } + } + + if (stringCode == StringCode::CodePoint) { + // Note: This condition must match tryAttachStringFromCodePoint to prevent + // failure loops. + masm.branch32(Assembler::Above, code, Imm32(unicode::NonBMPMax), + failure->label()); + } + + allocator.discardStack(masm); + + // We pre-allocate atoms for the first UNIT_STATIC_LIMIT characters. + // For code units larger than that, we must do a VM call. + Label vmCall; + masm.boundsCheck32PowerOfTwo(code, StaticStrings::UNIT_STATIC_LIMIT, &vmCall); + + masm.movePtr(ImmPtr(cx_->runtime()->staticStrings->unitStaticTable), scratch); + masm.loadPtr(BaseIndex(scratch, code, ScalePointer), scratch); + Label done; + masm.jump(&done); + + { + masm.bind(&vmCall); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.Push(code); + + if (stringCode == StringCode::CodeUnit) { + using Fn = JSLinearString* (*)(JSContext*, int32_t); + callVM(masm); + } else { + using Fn = JSString* (*)(JSContext*, int32_t); + callVM(masm); + } + + stubFrame.leave(masm); + masm.storeCallPointerResult(scratch); + } + + masm.bind(&done); + masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitStringFromCharCodeResult( + Int32OperandId codeId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + return emitStringFromCodeResult(codeId, StringCode::CodeUnit); +} + +bool BaselineCacheIRCompiler::emitStringFromCodePointResult( + Int32OperandId codeId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + return emitStringFromCodeResult(codeId, StringCode::CodePoint); +} + +bool BaselineCacheIRCompiler::emitMathRandomResult(uint32_t rngOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister64 scratch2(allocator, masm); + AutoAvailableFloatRegister scratchFloat(*this, FloatReg0); + + Address rngAddr(stubAddress(rngOffset)); + masm.loadPtr(rngAddr, scratch1); + + masm.randomDouble(scratch1, scratchFloat, scratch2, + output.valueReg().toRegister64()); + + if (js::SupportDifferentialTesting()) { + masm.loadConstantDouble(0.0, scratchFloat); + } + + masm.boxDouble(scratchFloat, output.valueReg(), scratchFloat); + return true; +} + +bool BaselineCacheIRCompiler::emitReflectGetPrototypeOfResult( + ObjOperandId objId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + Register obj = allocator.useRegister(masm, objId); + + allocator.discardStack(masm); + + MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1); + + masm.loadObjProto(obj, scratch); + + Label hasProto; + masm.branchPtr(Assembler::Above, scratch, ImmWord(1), &hasProto); + + // Call into the VM for lazy prototypes. + Label slow, done; + masm.branchPtr(Assembler::Equal, scratch, ImmWord(1), &slow); + + masm.moveValue(NullValue(), output.valueReg()); + masm.jump(&done); + + masm.bind(&hasProto); + masm.tagValue(JSVAL_TYPE_OBJECT, scratch, output.valueReg()); + masm.jump(&done); + + { + masm.bind(&slow); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.Push(obj); + + using Fn = bool (*)(JSContext*, HandleObject, MutableHandleValue); + callVM(masm); + + stubFrame.leave(masm); + } + + masm.bind(&done); + return true; +} + +bool BaselineCacheIRCompiler::emitHasClassResult(ObjOperandId objId, + uint32_t claspOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register obj = allocator.useRegister(masm, objId); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + + Address claspAddr(stubAddress(claspOffset)); + masm.loadObjClassUnsafe(obj, scratch); + masm.cmpPtrSet(Assembler::Equal, claspAddr, scratch.get(), scratch); + masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg()); + return true; +} + +void BaselineCacheIRCompiler::emitAtomizeString(Register str, Register temp, + Label* failure) { + Label isAtom; + masm.branchTest32(Assembler::NonZero, Address(str, JSString::offsetOfFlags()), + Imm32(JSString::ATOM_BIT), &isAtom); + { + LiveRegisterSet save(GeneralRegisterSet::Volatile(), + liveVolatileFloatRegs()); + masm.PushRegsInMask(save); + + using Fn = JSAtom* (*)(JSContext* cx, JSString* str); + masm.setupUnalignedABICall(temp); + masm.loadJSContext(temp); + masm.passABIArg(temp); + masm.passABIArg(str); + masm.callWithABI(); + masm.storeCallPointerResult(temp); + + LiveRegisterSet ignore; + ignore.add(temp); + masm.PopRegsInMaskIgnore(save, ignore); + + masm.branchPtr(Assembler::Equal, temp, ImmWord(0), failure); + masm.mov(temp, str); + } + masm.bind(&isAtom); +} + +bool BaselineCacheIRCompiler::emitSetHasStringResult(ObjOperandId setId, + StringOperandId strId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register set = allocator.useRegister(masm, setId); + Register str = allocator.useRegister(masm, strId); + + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + AutoScratchRegister scratch3(allocator, masm); + AutoScratchRegister scratch4(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + emitAtomizeString(str, scratch1, failure->label()); + masm.prepareHashString(str, scratch1, scratch2); + + masm.tagValue(JSVAL_TYPE_STRING, str, output.valueReg()); + masm.setObjectHasNonBigInt(set, output.valueReg(), scratch1, scratch2, + scratch3, scratch4); + masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitMapHasStringResult(ObjOperandId mapId, + StringOperandId strId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register map = allocator.useRegister(masm, mapId); + Register str = allocator.useRegister(masm, strId); + + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + AutoScratchRegister scratch3(allocator, masm); + AutoScratchRegister scratch4(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + emitAtomizeString(str, scratch1, failure->label()); + masm.prepareHashString(str, scratch1, scratch2); + + masm.tagValue(JSVAL_TYPE_STRING, str, output.valueReg()); + masm.mapObjectHasNonBigInt(map, output.valueReg(), scratch1, scratch2, + scratch3, scratch4); + masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitMapGetStringResult(ObjOperandId mapId, + StringOperandId strId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register map = allocator.useRegister(masm, mapId); + Register str = allocator.useRegister(masm, strId); + + AutoScratchRegister scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + AutoScratchRegister scratch3(allocator, masm); + AutoScratchRegister scratch4(allocator, masm); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + emitAtomizeString(str, scratch1, failure->label()); + masm.prepareHashString(str, scratch1, scratch2); + + masm.tagValue(JSVAL_TYPE_STRING, str, output.valueReg()); + masm.mapObjectGetNonBigInt(map, output.valueReg(), scratch1, + output.valueReg(), scratch2, scratch3, scratch4); + return true; +} + +bool BaselineCacheIRCompiler::emitCallNativeSetter( + ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId, + bool sameRealm, uint32_t nargsAndFlagsOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register receiver = allocator.useRegister(masm, receiverId); + Address setterAddr(stubAddress(setterOffset)); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Load the callee in the scratch register. + masm.loadPtr(setterAddr, scratch); + + masm.Push(val); + masm.Push(receiver); + masm.Push(scratch); + + using Fn = bool (*)(JSContext*, HandleFunction, HandleObject, HandleValue); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitCallScriptedSetterShared( + ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId, + bool sameRealm, uint32_t nargsAndFlagsOffset, + Maybe icScriptOffset) { + AutoScratchRegister callee(allocator, masm); + AutoScratchRegister scratch(allocator, masm); +#if defined(JS_CODEGEN_X86) + Register code = scratch; +#else + AutoScratchRegister code(allocator, masm); +#endif + + Register receiver = allocator.useRegister(masm, receiverId); + Address setterAddr(stubAddress(setterOffset)); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + + bool isInlined = icScriptOffset.isSome(); + + // First, load the callee. + masm.loadPtr(setterAddr, callee); + + if (isInlined) { + // If we are calling a trial-inlined setter, guard that the + // target has a BaselineScript. + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + masm.loadBaselineJitCodeRaw(callee, code, failure->label()); + } + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + if (!sameRealm) { + masm.switchToObjectRealm(callee, scratch); + } + + // Align the stack such that the JitFrameLayout is aligned on + // JitStackAlignment. + masm.alignJitStackBasedOnNArgs(1, /*countIncludesThis = */ false); + + // Setter is called with 1 argument, and |receiver| as thisv. Note that we use + // Push, not push, so that callJit will align the stack properly on ARM. + masm.Push(val); + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(receiver))); + + // Push callee. + masm.Push(callee); + + // Push frame descriptor. + masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, /* argc = */ 1); + + if (isInlined) { + // Store icScript in the context. + Address icScriptAddr(stubAddress(*icScriptOffset)); + masm.loadPtr(icScriptAddr, scratch); + masm.storeICScriptInJSContext(scratch); + } + + // Load the jitcode pointer. + if (isInlined) { + // On non-x86 platforms, this pointer is still in a register + // after guarding on it above. On x86, we don't have enough + // registers and have to reload it here. +#ifdef JS_CODEGEN_X86 + masm.loadBaselineJitCodeRaw(callee, code); +#endif + } else { + masm.loadJitCodeRaw(callee, code); + } + + // Handle arguments underflow. The rhs value is no longer needed and + // can be used as scratch. + Label noUnderflow; + Register scratch2 = val.scratchReg(); + masm.loadFunctionArgCount(callee, scratch2); + masm.branch32(Assembler::BelowOrEqual, scratch2, Imm32(1), &noUnderflow); + + // Call the arguments rectifier. + ArgumentsRectifierKind kind = isInlined + ? ArgumentsRectifierKind::TrialInlining + : ArgumentsRectifierKind::Normal; + TrampolinePtr argumentsRectifier = + cx_->runtime()->jitRuntime()->getArgumentsRectifier(kind); + masm.movePtr(argumentsRectifier, code); + + masm.bind(&noUnderflow); + masm.callJit(code); + + stubFrame.leave(masm); + + if (!sameRealm) { + masm.switchToBaselineFrameRealm(R1.scratchReg()); + } + + return true; +} + +bool BaselineCacheIRCompiler::emitCallScriptedSetter( + ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId, + bool sameRealm, uint32_t nargsAndFlagsOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Maybe icScriptOffset = mozilla::Nothing(); + return emitCallScriptedSetterShared(receiverId, setterOffset, rhsId, + sameRealm, nargsAndFlagsOffset, + icScriptOffset); +} + +bool BaselineCacheIRCompiler::emitCallInlinedSetter( + ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId, + uint32_t icScriptOffset, bool sameRealm, uint32_t nargsAndFlagsOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + return emitCallScriptedSetterShared(receiverId, setterOffset, rhsId, + sameRealm, nargsAndFlagsOffset, + mozilla::Some(icScriptOffset)); +} + +bool BaselineCacheIRCompiler::emitCallDOMSetter(ObjOperandId objId, + uint32_t jitInfoOffset, + ValOperandId rhsId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + Address jitInfoAddr(stubAddress(jitInfoOffset)); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Load the JSJitInfo in the scratch register. + masm.loadPtr(jitInfoAddr, scratch); + + masm.Push(val); + masm.Push(obj); + masm.Push(scratch); + + using Fn = bool (*)(JSContext*, const JSJitInfo*, HandleObject, HandleValue); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitCallSetArrayLength(ObjOperandId objId, + bool strict, + ValOperandId rhsId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.Push(Imm32(strict)); + masm.Push(val); + masm.Push(obj); + + using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitProxySet(ObjOperandId objId, + uint32_t idOffset, + ValOperandId rhsId, bool strict) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + Address idAddr(stubAddress(idOffset)); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Load the jsid in the scratch register. + masm.loadPtr(idAddr, scratch); + + masm.Push(Imm32(strict)); + masm.Push(val); + masm.Push(scratch); + masm.Push(obj); + + using Fn = bool (*)(JSContext*, HandleObject, HandleId, HandleValue, bool); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitProxySetByValue(ObjOperandId objId, + ValOperandId idId, + ValOperandId rhsId, + bool strict) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + ValueOperand idVal = allocator.useValueRegister(masm, idId); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + + allocator.discardStack(masm); + + // We need a scratch register but we don't have any registers available on + // x86, so temporarily store |obj| in the frame's scratch slot. + int scratchOffset = BaselineFrame::reverseOffsetOfScratchValue(); + masm.storePtr(obj, Address(baselineFrameReg(), scratchOffset)); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, obj); + + // Restore |obj|. Because we entered a stub frame we first have to load + // the original frame pointer. + masm.loadPtr(Address(FramePointer, 0), obj); + masm.loadPtr(Address(obj, scratchOffset), obj); + + masm.Push(Imm32(strict)); + masm.Push(val); + masm.Push(idVal); + masm.Push(obj); + + using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue, bool); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitCallAddOrUpdateSparseElementHelper( + ObjOperandId objId, Int32OperandId idId, ValOperandId rhsId, bool strict) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + Register id = allocator.useRegister(masm, idId); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.Push(Imm32(strict)); + masm.Push(val); + masm.Push(id); + masm.Push(obj); + + using Fn = bool (*)(JSContext* cx, Handle obj, int32_t int_id, + HandleValue v, bool strict); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitMegamorphicSetElement(ObjOperandId objId, + ValOperandId idId, + ValOperandId rhsId, + bool strict) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + ValueOperand idVal = allocator.useValueRegister(masm, idId); + ValueOperand val = allocator.useValueRegister(masm, rhsId); + +#ifdef JS_CODEGEN_X86 + allocator.discardStack(masm); + // We need a scratch register but we don't have any registers available on + // x86, so temporarily store |obj| in the frame's scratch slot. + int scratchOffset = BaselineFrame::reverseOffsetOfScratchValue(); + masm.storePtr(obj, Address(baselineFrameReg_, scratchOffset)); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, obj); + + // Restore |obj|. Because we entered a stub frame we first have to load + // the original frame pointer. + masm.loadPtr(Address(FramePointer, 0), obj); + masm.loadPtr(Address(obj, scratchOffset), obj); +#else + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); +#endif + + masm.Push(Imm32(strict)); + masm.Push(val); + masm.Push(idVal); + masm.Push(obj); + + using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue, bool); + callVM>(masm); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitReturnFromIC() { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + allocator.discardStack(masm); + if (JitOptions.enableICFramePointers) { + masm.pop(FramePointer); + } + EmitReturnFromIC(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitLoadArgumentFixedSlot(ValOperandId resultId, + uint8_t slotIndex) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + ValueOperand resultReg = allocator.defineValueRegister(masm, resultId); + Address addr = allocator.addressOf(masm, BaselineFrameSlot(slotIndex)); + masm.loadValue(addr, resultReg); + return true; +} + +bool BaselineCacheIRCompiler::emitLoadArgumentDynamicSlot(ValOperandId resultId, + Int32OperandId argcId, + uint8_t slotIndex) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + ValueOperand resultReg = allocator.defineValueRegister(masm, resultId); + Register argcReg = allocator.useRegister(masm, argcId); + BaseValueIndex addr = + allocator.addressOf(masm, argcReg, BaselineFrameSlot(slotIndex)); + masm.loadValue(addr, resultReg); + return true; +} + +bool BaselineCacheIRCompiler::emitGuardDOMExpandoMissingOrGuardShape( + ValOperandId expandoId, uint32_t shapeOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + ValueOperand val = allocator.useValueRegister(masm, expandoId); + AutoScratchRegister shapeScratch(allocator, masm); + AutoScratchRegister objScratch(allocator, masm); + Address shapeAddr(stubAddress(shapeOffset)); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + Label done; + masm.branchTestUndefined(Assembler::Equal, val, &done); + + masm.debugAssertIsObject(val); + masm.loadPtr(shapeAddr, shapeScratch); + masm.unboxObject(val, objScratch); + // The expando object is not used in this case, so we don't need Spectre + // mitigations. + masm.branchTestObjShapeNoSpectreMitigations(Assembler::NotEqual, objScratch, + shapeScratch, failure->label()); + + masm.bind(&done); + return true; +} + +bool BaselineCacheIRCompiler::emitLoadDOMExpandoValueGuardGeneration( + ObjOperandId objId, uint32_t expandoAndGenerationOffset, + uint32_t generationOffset, ValOperandId resultId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register obj = allocator.useRegister(masm, objId); + Address expandoAndGenerationAddr(stubAddress(expandoAndGenerationOffset)); + Address generationAddr(stubAddress(generationOffset)); + + AutoScratchRegister scratch(allocator, masm); + ValueOperand output = allocator.defineValueRegister(masm, resultId); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), scratch); + Address expandoAddr(scratch, + js::detail::ProxyReservedSlots::offsetOfPrivateSlot()); + + // Load the ExpandoAndGeneration* in the output scratch register and guard + // it matches the proxy's ExpandoAndGeneration. + masm.loadPtr(expandoAndGenerationAddr, output.scratchReg()); + masm.branchPrivatePtr(Assembler::NotEqual, expandoAddr, output.scratchReg(), + failure->label()); + + // Guard expandoAndGeneration->generation matches the expected generation. + masm.branch64( + Assembler::NotEqual, + Address(output.scratchReg(), ExpandoAndGeneration::offsetOfGeneration()), + generationAddr, scratch, failure->label()); + + // Load expandoAndGeneration->expando into the output Value register. + masm.loadValue( + Address(output.scratchReg(), ExpandoAndGeneration::offsetOfExpando()), + output); + return true; +} + +bool BaselineCacheIRCompiler::init(CacheKind kind) { + if (!allocator.init()) { + return false; + } + + size_t numInputs = writer_.numInputOperands(); + MOZ_ASSERT(numInputs == NumInputsForCacheKind(kind)); + + // Baseline passes the first 2 inputs in R0/R1, other Values are stored on + // the stack. + size_t numInputsInRegs = std::min(numInputs, size_t(2)); + AllocatableGeneralRegisterSet available = + BaselineICAvailableGeneralRegs(numInputsInRegs); + + switch (kind) { + case CacheKind::NewArray: + case CacheKind::NewObject: + case CacheKind::GetIntrinsic: + MOZ_ASSERT(numInputs == 0); + outputUnchecked_.emplace(R0); + break; + case CacheKind::GetProp: + case CacheKind::TypeOf: + case CacheKind::ToPropertyKey: + case CacheKind::GetIterator: + case CacheKind::OptimizeSpreadCall: + case CacheKind::ToBool: + case CacheKind::UnaryArith: + MOZ_ASSERT(numInputs == 1); + allocator.initInputLocation(0, R0); + outputUnchecked_.emplace(R0); + break; + case CacheKind::Compare: + case CacheKind::GetElem: + case CacheKind::GetPropSuper: + case CacheKind::In: + case CacheKind::HasOwn: + case CacheKind::CheckPrivateField: + case CacheKind::InstanceOf: + case CacheKind::BinaryArith: + MOZ_ASSERT(numInputs == 2); + allocator.initInputLocation(0, R0); + allocator.initInputLocation(1, R1); + outputUnchecked_.emplace(R0); + break; + case CacheKind::SetProp: + MOZ_ASSERT(numInputs == 2); + allocator.initInputLocation(0, R0); + allocator.initInputLocation(1, R1); + break; + case CacheKind::GetElemSuper: + MOZ_ASSERT(numInputs == 3); + allocator.initInputLocation(0, BaselineFrameSlot(0)); + allocator.initInputLocation(1, R1); + allocator.initInputLocation(2, R0); + outputUnchecked_.emplace(R0); + break; + case CacheKind::SetElem: + MOZ_ASSERT(numInputs == 3); + allocator.initInputLocation(0, R0); + allocator.initInputLocation(1, R1); + allocator.initInputLocation(2, BaselineFrameSlot(0)); + break; + case CacheKind::GetName: + case CacheKind::BindName: + MOZ_ASSERT(numInputs == 1); + allocator.initInputLocation(0, R0.scratchReg(), JSVAL_TYPE_OBJECT); +#if defined(JS_NUNBOX32) + // availableGeneralRegs can't know that GetName/BindName is only using + // the payloadReg and not typeReg on x86. + available.add(R0.typeReg()); +#endif + outputUnchecked_.emplace(R0); + break; + case CacheKind::Call: + MOZ_ASSERT(numInputs == 1); + allocator.initInputLocation(0, R0.scratchReg(), JSVAL_TYPE_INT32); +#if defined(JS_NUNBOX32) + // availableGeneralRegs can't know that Call is only using + // the payloadReg and not typeReg on x86. + available.add(R0.typeReg()); +#endif + outputUnchecked_.emplace(R0); + break; + case CacheKind::CloseIter: + MOZ_ASSERT(numInputs == 1); + allocator.initInputLocation(0, R0.scratchReg(), JSVAL_TYPE_OBJECT); +#if defined(JS_NUNBOX32) + // availableGeneralRegs can't know that CloseIter is only using + // the payloadReg and not typeReg on x86. + available.add(R0.typeReg()); +#endif + break; + } + + // Baseline doesn't allocate float registers so none of them are live. + liveFloatRegs_ = LiveFloatRegisterSet(FloatRegisterSet()); + + if (JitOptions.enableICFramePointers) { + baselineFrameReg_ = available.takeAny(); + } + + allocator.initAvailableRegs(available); + return true; +} + +static void ResetEnteredCounts(const ICEntry* icEntry) { + ICStub* stub = icEntry->firstStub(); + while (true) { + stub->resetEnteredCount(); + if (stub->isFallback()) { + return; + } + stub = stub->toCacheIRStub()->next(); + } +} + +static ICStubSpace* StubSpaceForStub(bool makesGCCalls, JSScript* script, + ICScript* icScript) { + if (makesGCCalls) { + return icScript->jitScriptStubSpace(); + } + return script->zone()->jitZone()->optimizedStubSpace(); +} + +static const uint32_t MaxFoldedShapes = 16; + +bool js::jit::TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback, + JSScript* script, ICScript* icScript) { + ICEntry* icEntry = icScript->icEntryForStub(fallback); + ICStub* entryStub = icEntry->firstStub(); + + // Don't fold unless there are at least two stubs. + if (entryStub == fallback) { + return true; + } + ICCacheIRStub* firstStub = entryStub->toCacheIRStub(); + if (firstStub->next()->isFallback()) { + return true; + } + + const uint8_t* firstStubData = firstStub->stubDataStart(); + const CacheIRStubInfo* stubInfo = firstStub->stubInfo(); + + // Check to see if: + // a) all of the stubs in this chain have the exact same code. + // b) all of the stubs have the same stub field data, except + // for a single GuardShape where they differ. + // c) at least one stub after the first has a non-zero entry count. + // + // If all of these conditions hold, then we generate a single stub + // that covers all the existing cases by replacing GuardShape with + // GuardMultipleShapes. + + uint32_t numActive = 0; + Maybe foldableFieldOffset; + RootedValue shape(cx); + RootedValueVector shapeList(cx); + + auto addShape = [&shapeList, cx](uintptr_t rawShape) -> bool { + Shape* shape = reinterpret_cast(rawShape); + if (cx->compartment() != shape->compartment()) { + return false; + } + if (!shapeList.append(PrivateGCThingValue(shape))) { + cx->recoverFromOutOfMemory(); + return false; + } + return true; + }; + + for (ICCacheIRStub* other = firstStub->nextCacheIR(); other; + other = other->nextCacheIR()) { + // Verify that the stubs share the same code. + if (other->stubInfo() != stubInfo) { + return true; + } + const uint8_t* otherStubData = other->stubDataStart(); + + if (other->enteredCount() > 0) { + numActive++; + } + + uint32_t fieldIndex = 0; + size_t offset = 0; + while (stubInfo->fieldType(fieldIndex) != StubField::Type::Limit) { + StubField::Type fieldType = stubInfo->fieldType(fieldIndex); + + if (StubField::sizeIsWord(fieldType)) { + uintptr_t firstRaw = stubInfo->getStubRawWord(firstStubData, offset); + uintptr_t otherRaw = stubInfo->getStubRawWord(otherStubData, offset); + + if (firstRaw != otherRaw) { + if (fieldType != StubField::Type::Shape) { + // Case 1: a field differs that is not a Shape. We only support + // folding GuardShape to GuardMultipleShapes. + return true; + } + if (foldableFieldOffset.isNothing()) { + // Case 2: this is the first field where the stub data differs. + foldableFieldOffset.emplace(offset); + if (!addShape(firstRaw) || !addShape(otherRaw)) { + return true; + } + } else if (*foldableFieldOffset == offset) { + // Case 3: this is the corresponding offset in a different stub. + if (!addShape(otherRaw)) { + return true; + } + } else { + // Case 4: we have found more than one field that differs. + return true; + } + } + } else { + MOZ_ASSERT(StubField::sizeIsInt64(fieldType)); + + // We do not support folding any ops with int64-sized fields. + if (stubInfo->getStubRawInt64(firstStubData, offset) != + stubInfo->getStubRawInt64(otherStubData, offset)) { + return true; + } + } + + offset += StubField::sizeInBytes(fieldType); + fieldIndex++; + } + + // We should never attach two completely identical stubs. + MOZ_ASSERT(foldableFieldOffset.isSome()); + } + + if (numActive == 0) { + return true; + } + + // Clone the CacheIR, replacing GuardShape with GuardMultipleShapes. + CacheIRWriter writer(cx); + CacheIRReader reader(stubInfo); + CacheIRCloner cloner(firstStub); + + // Initialize the operands. + CacheKind cacheKind = stubInfo->kind(); + for (uint32_t i = 0; i < NumInputsForCacheKind(cacheKind); i++) { + writer.setInputOperandId(i); + } + + bool success = false; + while (reader.more()) { + CacheOp op = reader.readOp(); + switch (op) { + case CacheOp::GuardShape: { + ObjOperandId objId = reader.objOperandId(); + uint32_t shapeOffset = reader.stubOffset(); + if (shapeOffset == *foldableFieldOffset) { + // Ensure that the allocation of the ListObject doesn't trigger a GC + // and free the stubInfo we're currently reading. Note that + // AutoKeepJitScripts isn't sufficient, because optimized stubs can be + // discarded even if the JitScript is preserved. + gc::AutoSuppressGC suppressGC(cx); + + Rooted shapeObj(cx, ListObject::create(cx)); + if (!shapeObj) { + return false; + } + for (uint32_t i = 0; i < shapeList.length(); i++) { + if (!shapeObj->append(cx, shapeList[i])) { + cx->recoverFromOutOfMemory(); + return false; + } + } + + writer.guardMultipleShapes(objId, shapeObj); + success = true; + } else { + Shape* shape = stubInfo->getStubField(firstStub, shapeOffset); + writer.guardShape(objId, shape); + } + break; + } + default: + cloner.cloneOp(op, reader, writer); + break; + } + } + if (!success) { + // If the shape field that differed was not part of a GuardShape, + // we can't fold these stubs together. + return true; + } + + // Replace the existing stubs with the new folded stub. + fallback->discardStubs(cx, icEntry); + + ICAttachResult result = AttachBaselineCacheIRStub( + cx, writer, cacheKind, script, icScript, fallback, "StubFold"); + if (result == ICAttachResult::OOM) { + ReportOutOfMemory(cx); + return false; + } + MOZ_ASSERT(result == ICAttachResult::Attached); + + fallback->setHasFoldedStub(); + return true; +} + +static bool AddToFoldedStub(JSContext* cx, const CacheIRWriter& writer, + ICScript* icScript, ICFallbackStub* fallback) { + ICEntry* icEntry = icScript->icEntryForStub(fallback); + ICStub* entryStub = icEntry->firstStub(); + + // We only update folded stubs if they're the only stub in the IC. + if (entryStub == fallback) { + return false; + } + ICCacheIRStub* stub = entryStub->toCacheIRStub(); + if (!stub->next()->isFallback()) { + return false; + } + + const CacheIRStubInfo* stubInfo = stub->stubInfo(); + const uint8_t* stubData = stub->stubDataStart(); + + Maybe shapeFieldOffset; + RootedValue newShape(cx); + Rooted foldedShapes(cx); + + CacheIRReader stubReader(stubInfo); + CacheIRReader newReader(writer); + while (newReader.more() && stubReader.more()) { + CacheOp newOp = newReader.readOp(); + CacheOp stubOp = stubReader.readOp(); + switch (stubOp) { + case CacheOp::GuardMultipleShapes: { + // Check that the new stub has a corresponding GuardShape. + if (newOp != CacheOp::GuardShape) { + return false; + } + + // Check that the object being guarded is the same. + if (newReader.objOperandId() != stubReader.objOperandId()) { + return false; + } + + // Check that the field offset is the same. + uint32_t newShapeOffset = newReader.stubOffset(); + uint32_t stubShapesOffset = stubReader.stubOffset(); + if (newShapeOffset != stubShapesOffset) { + return false; + } + MOZ_ASSERT(shapeFieldOffset.isNothing()); + shapeFieldOffset.emplace(newShapeOffset); + + // Get the shape from the new stub + StubField shapeField = + writer.readStubField(newShapeOffset, StubField::Type::Shape); + Shape* shape = reinterpret_cast(shapeField.asWord()); + newShape = PrivateGCThingValue(shape); + + // Get the shape array from the old stub. + JSObject* shapeList = + stubInfo->getStubField(stub, stubShapesOffset); + foldedShapes = &shapeList->as(); + MOZ_ASSERT(foldedShapes->compartment() == shape->compartment()); + break; + } + default: { + // Check that the op is the same. + if (newOp != stubOp) { + return false; + } + + // Check that the arguments are the same. + uint32_t argLength = CacheIROpInfos[size_t(newOp)].argLength; + for (uint32_t i = 0; i < argLength; i++) { + if (newReader.readByte() != stubReader.readByte()) { + return false; + } + } + } + } + } + + MOZ_ASSERT(shapeFieldOffset.isSome()); + + // Check to verify that all the other stub fields are the same. + if (!writer.stubDataEqualsIgnoring(stubData, *shapeFieldOffset)) { + return false; + } + + // Limit the maximum number of shapes we will add before giving up. + if (foldedShapes->length() == MaxFoldedShapes) { + return false; + } + + if (!foldedShapes->append(cx, newShape)) { + cx->recoverFromOutOfMemory(); + return false; + } + + return true; +} + +ICAttachResult js::jit::AttachBaselineCacheIRStub( + JSContext* cx, const CacheIRWriter& writer, CacheKind kind, + JSScript* outerScript, ICScript* icScript, ICFallbackStub* stub, + const char* name) { + // We shouldn't GC or report OOM (or any other exception) here. + AutoAssertNoPendingException aanpe(cx); + JS::AutoCheckCannotGC nogc; + + if (writer.tooLarge()) { + return ICAttachResult::TooLarge; + } + if (writer.oom()) { + return ICAttachResult::OOM; + } + MOZ_ASSERT(!writer.failed()); + + // Just a sanity check: the caller should ensure we don't attach an + // unlimited number of stubs. +#ifdef DEBUG + static const size_t MaxOptimizedCacheIRStubs = 16; + MOZ_ASSERT(stub->numOptimizedStubs() < MaxOptimizedCacheIRStubs); +#endif + + constexpr uint32_t stubDataOffset = sizeof(ICCacheIRStub); + static_assert(stubDataOffset % sizeof(uint64_t) == 0, + "Stub fields must be aligned"); + + JitZone* jitZone = cx->zone()->jitZone(); + + // Check if we already have JitCode for this stub. + CacheIRStubInfo* stubInfo; + CacheIRStubKey::Lookup lookup(kind, ICStubEngine::Baseline, + writer.codeStart(), writer.codeLength()); + JitCode* code = jitZone->getBaselineCacheIRStubCode(lookup, &stubInfo); + if (!code) { + // We have to generate stub code. + TempAllocator temp(&cx->tempLifoAlloc()); + JitContext jctx(cx); + BaselineCacheIRCompiler comp(cx, temp, writer, stubDataOffset); + if (!comp.init(kind)) { + return ICAttachResult::OOM; + } + + code = comp.compile(); + if (!code) { + return ICAttachResult::OOM; + } + + comp.perfSpewer().saveProfile(code, name); + + // Allocate the shared CacheIRStubInfo. Note that the + // putBaselineCacheIRStubCode call below will transfer ownership + // to the stub code HashMap, so we don't have to worry about freeing + // it below. + MOZ_ASSERT(!stubInfo); + stubInfo = + CacheIRStubInfo::New(kind, ICStubEngine::Baseline, comp.makesGCCalls(), + stubDataOffset, writer); + if (!stubInfo) { + return ICAttachResult::OOM; + } + + CacheIRStubKey key(stubInfo); + if (!jitZone->putBaselineCacheIRStubCode(lookup, key, code)) { + return ICAttachResult::OOM; + } + } + + MOZ_ASSERT(code); + MOZ_ASSERT(stubInfo); + MOZ_ASSERT(stubInfo->stubDataSize() == writer.stubDataSize()); + + ICEntry* icEntry = icScript->icEntryForStub(stub); + + // Ensure we don't attach duplicate stubs. This can happen if a stub failed + // for some reason and the IR generator doesn't check for exactly the same + // conditions. + for (ICStub* iter = icEntry->firstStub(); iter != stub; + iter = iter->toCacheIRStub()->next()) { + auto otherStub = iter->toCacheIRStub(); + if (otherStub->stubInfo() != stubInfo) { + continue; + } + if (!writer.stubDataEquals(otherStub->stubDataStart())) { + continue; + } + + // We found a stub that's exactly the same as the stub we're about to + // attach. Just return nullptr, the caller should do nothing in this + // case. + JitSpew(JitSpew_BaselineICFallback, + "Tried attaching identical stub for (%s:%u:%u)", + outerScript->filename(), outerScript->lineno(), + outerScript->column()); + return ICAttachResult::DuplicateStub; + } + + // Try including this case in an existing folded stub. + if (stub->hasFoldedStub() && AddToFoldedStub(cx, writer, icScript, stub)) { + // Instead of adding a new stub, we have added a new case to an + // existing folded stub. We do not have to invalidate Warp, + // because the ListObject that stores the cases is shared between + // baseline and Warp. Reset the entered count for the fallback + // stub so that we can still transpile, and reset the bailout + // counter if we have already been transpiled. + stub->resetEnteredCount(); + JSScript* owningScript = nullptr; + if (cx->zone()->jitZone()->hasStubFoldingBailoutData(outerScript)) { + owningScript = cx->zone()->jitZone()->stubFoldingBailoutParent(); + } else { + owningScript = icScript->isInlined() + ? icScript->inliningRoot()->owningScript() + : outerScript; + } + cx->zone()->jitZone()->clearStubFoldingBailoutData(); + if (stub->usedByTranspiler() && owningScript->hasIonScript()) { + owningScript->ionScript()->resetNumFixableBailouts(); + } + return ICAttachResult::Attached; + } + + // Time to allocate and attach a new stub. + + size_t bytesNeeded = stubInfo->stubDataOffset() + stubInfo->stubDataSize(); + + ICStubSpace* stubSpace = + StubSpaceForStub(stubInfo->makesGCCalls(), outerScript, icScript); + void* newStubMem = stubSpace->alloc(bytesNeeded); + if (!newStubMem) { + return ICAttachResult::OOM; + } + + // Resetting the entered counts on the IC chain makes subsequent reasoning + // about the chain much easier. + ResetEnteredCounts(icEntry); + + switch (stub->trialInliningState()) { + case TrialInliningState::Initial: + case TrialInliningState::Candidate: + stub->setTrialInliningState(writer.trialInliningState()); + break; + case TrialInliningState::MonomorphicInlined: + case TrialInliningState::Inlined: + stub->setTrialInliningState(TrialInliningState::Failure); + break; + case TrialInliningState::Failure: + break; + } + + auto newStub = new (newStubMem) ICCacheIRStub(code, stubInfo); + writer.copyStubData(newStub->stubDataStart()); + newStub->setTypeData(writer.typeData()); + stub->addNewStub(icEntry, newStub); + return ICAttachResult::Attached; +} + +uint8_t* ICCacheIRStub::stubDataStart() { + return reinterpret_cast(this) + stubInfo_->stubDataOffset(); +} + +bool BaselineCacheIRCompiler::emitCallStringObjectConcatResult( + ValOperandId lhsId, ValOperandId rhsId) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + ValueOperand lhs = allocator.useValueRegister(masm, lhsId); + ValueOperand rhs = allocator.useValueRegister(masm, rhsId); + + AutoScratchRegister scratch(allocator, masm); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.pushValue(rhs); + masm.pushValue(lhs); + + using Fn = bool (*)(JSContext*, HandleValue, HandleValue, MutableHandleValue); + callVM(masm); + + stubFrame.leave(masm); + return true; +} + +// The value of argc entering the call IC is not always the value of +// argc entering the callee. (For example, argc for a spread call IC +// is always 1, but argc for the callee is the length of the array.) +// In these cases, we update argc as part of the call op itself, to +// avoid modifying input operands while it is still possible to fail a +// guard. We also limit callee argc to a reasonable value to avoid +// blowing the stack limit. +bool BaselineCacheIRCompiler::updateArgc(CallFlags flags, Register argcReg, + Register scratch) { + CallFlags::ArgFormat format = flags.getArgFormat(); + switch (format) { + case CallFlags::Standard: + // Standard calls have no extra guards, and argc is already correct. + return true; + case CallFlags::FunCall: + // fun_call has no extra guards, and argc will be corrected in + // pushFunCallArguments. + return true; + case CallFlags::FunApplyNullUndefined: + // argc must be 0 if null or undefined is passed as second argument to + // |apply|. + masm.move32(Imm32(0), argcReg); + return true; + default: + break; + } + + // We need to guard the length of the arguments. + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + // Load callee argc into scratch. + switch (flags.getArgFormat()) { + case CallFlags::Spread: + case CallFlags::FunApplyArray: { + // Load the length of the elements. + BaselineFrameSlot slot(flags.isConstructing()); + masm.unboxObject(allocator.addressOf(masm, slot), scratch); + masm.loadPtr(Address(scratch, NativeObject::offsetOfElements()), scratch); + masm.load32(Address(scratch, ObjectElements::offsetOfLength()), scratch); + break; + } + case CallFlags::FunApplyArgsObj: { + // Load the arguments object length. + BaselineFrameSlot slot(0); + masm.unboxObject(allocator.addressOf(masm, slot), scratch); + masm.loadArgumentsObjectLength(scratch, scratch, failure->label()); + break; + } + default: + MOZ_CRASH("Unknown arg format"); + } + + // Ensure that callee argc does not exceed the limit. + masm.branch32(Assembler::Above, scratch, Imm32(JIT_ARGS_LENGTH_MAX), + failure->label()); + + // We're past the final guard. Update argc with the new value. + masm.move32(scratch, argcReg); + + return true; +} + +void BaselineCacheIRCompiler::pushArguments(Register argcReg, + Register calleeReg, + Register scratch, Register scratch2, + CallFlags flags, uint32_t argcFixed, + bool isJitCall) { + switch (flags.getArgFormat()) { + case CallFlags::Standard: + pushStandardArguments(argcReg, scratch, scratch2, argcFixed, isJitCall, + flags.isConstructing()); + break; + case CallFlags::Spread: + pushArrayArguments(argcReg, scratch, scratch2, isJitCall, + flags.isConstructing()); + break; + case CallFlags::FunCall: + pushFunCallArguments(argcReg, calleeReg, scratch, scratch2, argcFixed, + isJitCall); + break; + case CallFlags::FunApplyArgsObj: + pushFunApplyArgsObj(argcReg, calleeReg, scratch, scratch2, isJitCall); + break; + case CallFlags::FunApplyArray: + pushArrayArguments(argcReg, scratch, scratch2, isJitCall, + /*isConstructing =*/false); + break; + case CallFlags::FunApplyNullUndefined: + pushFunApplyNullUndefinedArguments(calleeReg, isJitCall); + break; + default: + MOZ_CRASH("Invalid arg format"); + } +} + +void BaselineCacheIRCompiler::pushStandardArguments( + Register argcReg, Register scratch, Register scratch2, uint32_t argcFixed, + bool isJitCall, bool isConstructing) { + MOZ_ASSERT(enteredStubFrame_); + + // The arguments to the call IC are pushed on the stack left-to-right. + // Our calling conventions want them right-to-left in the callee, so + // we duplicate them on the stack in reverse order. + + int additionalArgc = 1 + !isJitCall + isConstructing; + if (argcFixed < MaxUnrolledArgCopy) { +#ifdef DEBUG + Label ok; + masm.branch32(Assembler::Equal, argcReg, Imm32(argcFixed), &ok); + masm.assumeUnreachable("Invalid argcFixed value"); + masm.bind(&ok); +#endif + + size_t realArgc = argcFixed + additionalArgc; + + if (isJitCall) { + masm.alignJitStackBasedOnNArgs(realArgc, /*countIncludesThis = */ true); + } + + for (size_t i = 0; i < realArgc; ++i) { + masm.pushValue(Address( + FramePointer, BaselineStubFrameLayout::Size() + i * sizeof(Value))); + } + } else { + MOZ_ASSERT(argcFixed == MaxUnrolledArgCopy); + + // argPtr initially points to the last argument. Skip the stub frame. + Register argPtr = scratch2; + Address argAddress(FramePointer, BaselineStubFrameLayout::Size()); + masm.computeEffectiveAddress(argAddress, argPtr); + + // countReg contains the total number of arguments to copy. + // In addition to the actual arguments, we have to copy hidden arguments. + // We always have to copy |this|. + // If we are constructing, we have to copy |newTarget|. + // If we are not a jit call, we have to copy |callee|. + // We use a scratch register to avoid clobbering argc, which is an input + // reg. + Register countReg = scratch; + masm.move32(argcReg, countReg); + masm.add32(Imm32(additionalArgc), countReg); + + // Align the stack such that the JitFrameLayout is aligned on the + // JitStackAlignment. + if (isJitCall) { + masm.alignJitStackBasedOnNArgs(countReg, /*countIncludesThis = */ true); + } + + // Push all values, starting at the last one. + Label loop, done; + masm.branchTest32(Assembler::Zero, countReg, countReg, &done); + masm.bind(&loop); + { + masm.pushValue(Address(argPtr, 0)); + masm.addPtr(Imm32(sizeof(Value)), argPtr); + + masm.branchSub32(Assembler::NonZero, Imm32(1), countReg, &loop); + } + masm.bind(&done); + } +} + +void BaselineCacheIRCompiler::pushArrayArguments(Register argcReg, + Register scratch, + Register scratch2, + bool isJitCall, + bool isConstructing) { + MOZ_ASSERT(enteredStubFrame_); + + // Pull the array off the stack before aligning. + Register startReg = scratch; + size_t arrayOffset = + (isConstructing * sizeof(Value)) + BaselineStubFrameLayout::Size(); + masm.unboxObject(Address(FramePointer, arrayOffset), startReg); + masm.loadPtr(Address(startReg, NativeObject::offsetOfElements()), startReg); + + // Align the stack such that the JitFrameLayout is aligned on the + // JitStackAlignment. + if (isJitCall) { + Register alignReg = argcReg; + if (isConstructing) { + // If we are constructing, we must take newTarget into account. + alignReg = scratch2; + masm.computeEffectiveAddress(Address(argcReg, 1), alignReg); + } + masm.alignJitStackBasedOnNArgs(alignReg, /*countIncludesThis =*/false); + } + + // Push newTarget, if necessary + if (isConstructing) { + masm.pushValue(Address(FramePointer, BaselineStubFrameLayout::Size())); + } + + // Push arguments: set up endReg to point to &array[argc] + Register endReg = scratch2; + BaseValueIndex endAddr(startReg, argcReg); + masm.computeEffectiveAddress(endAddr, endReg); + + // Copying pre-decrements endReg by 8 until startReg is reached + Label copyDone; + Label copyStart; + masm.bind(©Start); + masm.branchPtr(Assembler::Equal, endReg, startReg, ©Done); + masm.subPtr(Imm32(sizeof(Value)), endReg); + masm.pushValue(Address(endReg, 0)); + masm.jump(©Start); + masm.bind(©Done); + + // Push |this|. + size_t thisvOffset = + BaselineStubFrameLayout::Size() + (1 + isConstructing) * sizeof(Value); + masm.pushValue(Address(FramePointer, thisvOffset)); + + // Push |callee| if needed. + if (!isJitCall) { + size_t calleeOffset = + BaselineStubFrameLayout::Size() + (2 + isConstructing) * sizeof(Value); + masm.pushValue(Address(FramePointer, calleeOffset)); + } +} + +void BaselineCacheIRCompiler::pushFunApplyNullUndefinedArguments( + Register calleeReg, bool isJitCall) { + // argc is already set to 0, so we just have to push |this| and (for native + // calls) the callee. + + MOZ_ASSERT(enteredStubFrame_); + + // Align the stack such that the JitFrameLayout is aligned on the + // JitStackAlignment. + if (isJitCall) { + masm.alignJitStackBasedOnNArgs(0, /*countIncludesThis =*/false); + } + + // Push |this|. + size_t thisvOffset = BaselineStubFrameLayout::Size() + 1 * sizeof(Value); + masm.pushValue(Address(FramePointer, thisvOffset)); + + // Push |callee| if needed. + if (!isJitCall) { + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(calleeReg))); + } +} + +void BaselineCacheIRCompiler::pushFunCallArguments( + Register argcReg, Register calleeReg, Register scratch, Register scratch2, + uint32_t argcFixed, bool isJitCall) { + if (argcFixed == 0) { + if (isJitCall) { + // Align the stack to 0 args. + masm.alignJitStackBasedOnNArgs(0, /*countIncludesThis = */ false); + } + + // Store the new |this|. + masm.pushValue(UndefinedValue()); + + // Store |callee| if needed. + if (!isJitCall) { + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(calleeReg))); + } + } else if (argcFixed < MaxUnrolledArgCopy) { + // See below for why we subtract 1 from argcFixed. + argcFixed -= 1; + masm.sub32(Imm32(1), argcReg); + pushStandardArguments(argcReg, scratch, scratch2, argcFixed, isJitCall, + /*isConstructing =*/false); + } else { + Label zeroArgs, done; + masm.branchTest32(Assembler::Zero, argcReg, argcReg, &zeroArgs); + + // When we call fun_call, the stack looks like the left column (note + // that newTarget will not be present, because fun_call cannot be a + // constructor call): + // + // ***Arguments to fun_call*** + // callee (fun_call) ***Arguments to target*** + // this (target function) -----> callee + // arg0 (this of target) -----> this + // arg1 (arg0 of target) -----> arg0 + // argN (argN-1 of target) -----> arg1 + // + // As demonstrated in the right column, this is exactly what we need + // the stack to look like when calling pushStandardArguments for target, + // except with one more argument. If we subtract 1 from argc, + // everything works out correctly. + masm.sub32(Imm32(1), argcReg); + + pushStandardArguments(argcReg, scratch, scratch2, argcFixed, isJitCall, + /*isConstructing =*/false); + + masm.jump(&done); + masm.bind(&zeroArgs); + + // The exception is the case where argc == 0: + // + // ***Arguments to fun_call*** + // callee (fun_call) ***Arguments to target*** + // this (target function) -----> callee + // -----> this + // + // In this case, we push |undefined| for |this|. + + if (isJitCall) { + // Align the stack to 0 args. + masm.alignJitStackBasedOnNArgs(0, /*countIncludesThis = */ false); + } + + // Store the new |this|. + masm.pushValue(UndefinedValue()); + + // Store |callee| if needed. + if (!isJitCall) { + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(calleeReg))); + } + + masm.bind(&done); + } +} + +void BaselineCacheIRCompiler::pushFunApplyArgsObj(Register argcReg, + Register calleeReg, + Register scratch, + Register scratch2, + bool isJitCall) { + MOZ_ASSERT(enteredStubFrame_); + + // Load the arguments object off the stack before aligning. + Register argsReg = scratch; + masm.unboxObject(Address(FramePointer, BaselineStubFrameLayout::Size()), + argsReg); + + // Align the stack such that the JitFrameLayout is aligned on the + // JitStackAlignment. + if (isJitCall) { + masm.alignJitStackBasedOnNArgs(argcReg, /*countIncludesThis =*/false); + } + + // Load ArgumentsData. + masm.loadPrivate(Address(argsReg, ArgumentsObject::getDataSlotOffset()), + argsReg); + + // We push the arguments onto the stack last-to-first. + // Compute the bounds of the arguments array. + Register currReg = scratch2; + Address argsStartAddr(argsReg, ArgumentsData::offsetOfArgs()); + masm.computeEffectiveAddress(argsStartAddr, argsReg); + BaseValueIndex argsEndAddr(argsReg, argcReg); + masm.computeEffectiveAddress(argsEndAddr, currReg); + + // Loop until all arguments have been pushed. + Label done, loop; + masm.bind(&loop); + masm.branchPtr(Assembler::Equal, currReg, argsReg, &done); + masm.subPtr(Imm32(sizeof(Value)), currReg); + + Address currArgAddr(currReg, 0); +#ifdef DEBUG + // Arguments are forwarded to the call object if they are closed over. + // In this case, OVERRIDDEN_ELEMENTS_BIT should be set. + Label notForwarded; + masm.branchTestMagic(Assembler::NotEqual, currArgAddr, ¬Forwarded); + masm.assumeUnreachable("Should have checked for overridden elements"); + masm.bind(¬Forwarded); +#endif + masm.pushValue(currArgAddr); + + masm.jump(&loop); + masm.bind(&done); + + // Push arg0 as |this| for call + masm.pushValue( + Address(FramePointer, BaselineStubFrameLayout::Size() + sizeof(Value))); + + // Push |callee| if needed. + if (!isJitCall) { + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(calleeReg))); + } +} + +void BaselineCacheIRCompiler::pushBoundFunctionArguments( + Register argcReg, Register calleeReg, Register scratch, Register scratch2, + CallFlags flags, uint32_t numBoundArgs, bool isJitCall) { + bool isConstructing = flags.isConstructing(); + uint32_t additionalArgc = 1 + isConstructing; // |this| and newTarget + + // Calculate total number of Values to push. + Register countReg = scratch; + masm.computeEffectiveAddress(Address(argcReg, numBoundArgs + additionalArgc), + countReg); + + // Align the stack such that the JitFrameLayout is aligned on the + // JitStackAlignment. + if (isJitCall) { + masm.alignJitStackBasedOnNArgs(countReg, /*countIncludesThis = */ true); + } + + if (isConstructing) { + // Push the bound function's target as newTarget. + Address boundTarget(calleeReg, BoundFunctionObject::offsetOfTargetSlot()); + masm.pushValue(boundTarget); + } + + // Ensure argPtr initially points to the last argument. Skip the stub frame. + Register argPtr = scratch2; + Address argAddress(FramePointer, BaselineStubFrameLayout::Size()); + if (isConstructing) { + // Skip newTarget. + argAddress.offset += sizeof(Value); + } + masm.computeEffectiveAddress(argAddress, argPtr); + + // Push all supplied arguments, starting at the last one. + Label loop, done; + masm.branchTest32(Assembler::Zero, argcReg, argcReg, &done); + masm.move32(argcReg, countReg); + masm.bind(&loop); + { + masm.pushValue(Address(argPtr, 0)); + masm.addPtr(Imm32(sizeof(Value)), argPtr); + + masm.branchSub32(Assembler::NonZero, Imm32(1), countReg, &loop); + } + masm.bind(&done); + + // Push the bound arguments, starting at the last one. + constexpr size_t inlineArgsOffset = + BoundFunctionObject::offsetOfFirstInlineBoundArg(); + if (numBoundArgs <= BoundFunctionObject::MaxInlineBoundArgs) { + for (size_t i = 0; i < numBoundArgs; i++) { + size_t argIndex = numBoundArgs - i - 1; + Address argAddr(calleeReg, inlineArgsOffset + argIndex * sizeof(Value)); + masm.pushValue(argAddr); + } + } else { + masm.unboxObject(Address(calleeReg, inlineArgsOffset), scratch); + masm.loadPtr(Address(scratch, NativeObject::offsetOfElements()), scratch); + for (size_t i = 0; i < numBoundArgs; i++) { + size_t argIndex = numBoundArgs - i - 1; + Address argAddr(scratch, argIndex * sizeof(Value)); + masm.pushValue(argAddr); + } + } + + if (isConstructing) { + // Push the |this| Value. This is either the object we allocated or the + // JS_UNINITIALIZED_LEXICAL magic value. It's stored in the BaselineFrame, + // so skip past the stub frame, (unbound) arguments and newTarget. + BaseValueIndex thisAddress(FramePointer, argcReg, + BaselineStubFrameLayout::Size() + sizeof(Value)); + masm.pushValue(thisAddress, scratch); + } else { + // Push the bound |this|. + Address boundThis(calleeReg, BoundFunctionObject::offsetOfBoundThisSlot()); + masm.pushValue(boundThis); + } +} + +bool BaselineCacheIRCompiler::emitCallNativeShared( + NativeCallType callType, ObjOperandId calleeId, Int32OperandId argcId, + CallFlags flags, uint32_t argcFixed, Maybe ignoresReturnValue, + Maybe targetOffset) { + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + AutoScratchRegister scratch2(allocator, masm); + + Register calleeReg = allocator.useRegister(masm, calleeId); + Register argcReg = allocator.useRegister(masm, argcId); + + bool isConstructing = flags.isConstructing(); + bool isSameRealm = flags.isSameRealm(); + + if (!updateArgc(flags, argcReg, scratch)) { + return false; + } + + allocator.discardStack(masm); + + // Push a stub frame so that we can perform a non-tail call. + // Note that this leaves the return address in TailCallReg. + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + if (!isSameRealm) { + masm.switchToObjectRealm(calleeReg, scratch); + } + + pushArguments(argcReg, calleeReg, scratch, scratch2, flags, argcFixed, + /*isJitCall =*/false); + + // Native functions have the signature: + // + // bool (*)(JSContext*, unsigned, Value* vp) + // + // Where vp[0] is space for callee/return value, vp[1] is |this|, and vp[2] + // onward are the function arguments. + + // Initialize vp. + masm.moveStackPtrTo(scratch2.get()); + + // Construct a native exit frame. + masm.push(argcReg); + + masm.pushFrameDescriptor(FrameType::BaselineStub); + masm.push(ICTailCallReg); + masm.push(FramePointer); + masm.loadJSContext(scratch); + masm.enterFakeExitFrameForNative(scratch, scratch, isConstructing); + + // Execute call. + masm.setupUnalignedABICall(scratch); + masm.loadJSContext(scratch); + masm.passABIArg(scratch); + masm.passABIArg(argcReg); + masm.passABIArg(scratch2); + + switch (callType) { + case NativeCallType::Native: { +#ifdef JS_SIMULATOR + // The simulator requires VM calls to be redirected to a special + // swi instruction to handle them, so we store the redirected + // pointer in the stub and use that instead of the original one. + // (See CacheIRWriter::callNativeFunction.) + Address redirectedAddr(stubAddress(*targetOffset)); + masm.callWithABI(redirectedAddr); +#else + if (*ignoresReturnValue) { + masm.loadPrivate( + Address(calleeReg, JSFunction::offsetOfJitInfoOrScript()), + calleeReg); + masm.callWithABI( + Address(calleeReg, JSJitInfo::offsetOfIgnoresReturnValueNative())); + } else { + // This depends on the native function pointer being stored unchanged as + // a PrivateValue. + masm.callWithABI(Address(calleeReg, JSFunction::offsetOfNativeOrEnv())); + } +#endif + } break; + case NativeCallType::ClassHook: { + Address nativeAddr(stubAddress(*targetOffset)); + masm.callWithABI(nativeAddr); + } break; + } + + // Test for failure. + masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel()); + + // Load the return value. + masm.loadValue( + Address(masm.getStackPointer(), NativeExitFrameLayout::offsetOfResult()), + output.valueReg()); + + stubFrame.leave(masm); + + if (!isSameRealm) { + masm.switchToBaselineFrameRealm(scratch2); + } + + return true; +} + +#ifdef JS_SIMULATOR +bool BaselineCacheIRCompiler::emitCallNativeFunction(ObjOperandId calleeId, + Int32OperandId argcId, + CallFlags flags, + uint32_t argcFixed, + uint32_t targetOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Maybe ignoresReturnValue; + Maybe targetOffset_ = mozilla::Some(targetOffset); + return emitCallNativeShared(NativeCallType::Native, calleeId, argcId, flags, + argcFixed, ignoresReturnValue, targetOffset_); +} + +bool BaselineCacheIRCompiler::emitCallDOMFunction( + ObjOperandId calleeId, Int32OperandId argcId, ObjOperandId thisObjId, + CallFlags flags, uint32_t argcFixed, uint32_t targetOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Maybe ignoresReturnValue; + Maybe targetOffset_ = mozilla::Some(targetOffset); + return emitCallNativeShared(NativeCallType::Native, calleeId, argcId, flags, + argcFixed, ignoresReturnValue, targetOffset_); +} +#else +bool BaselineCacheIRCompiler::emitCallNativeFunction(ObjOperandId calleeId, + Int32OperandId argcId, + CallFlags flags, + uint32_t argcFixed, + bool ignoresReturnValue) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Maybe ignoresReturnValue_ = mozilla::Some(ignoresReturnValue); + Maybe targetOffset; + return emitCallNativeShared(NativeCallType::Native, calleeId, argcId, flags, + argcFixed, ignoresReturnValue_, targetOffset); +} + +bool BaselineCacheIRCompiler::emitCallDOMFunction(ObjOperandId calleeId, + Int32OperandId argcId, + ObjOperandId thisObjId, + CallFlags flags, + uint32_t argcFixed) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Maybe ignoresReturnValue = mozilla::Some(false); + Maybe targetOffset; + return emitCallNativeShared(NativeCallType::Native, calleeId, argcId, flags, + argcFixed, ignoresReturnValue, targetOffset); +} +#endif + +bool BaselineCacheIRCompiler::emitCallClassHook(ObjOperandId calleeId, + Int32OperandId argcId, + CallFlags flags, + uint32_t argcFixed, + uint32_t targetOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Maybe ignoresReturnValue; + Maybe targetOffset_ = mozilla::Some(targetOffset); + return emitCallNativeShared(NativeCallType::ClassHook, calleeId, argcId, + flags, argcFixed, ignoresReturnValue, + targetOffset_); +} + +// Helper function for loading call arguments from the stack. Loads +// and unboxes an object from a specific slot. +void BaselineCacheIRCompiler::loadStackObject(ArgumentKind kind, + CallFlags flags, Register argcReg, + Register dest) { + MOZ_ASSERT(enteredStubFrame_); + + bool addArgc = false; + int32_t slotIndex = GetIndexOfArgument(kind, flags, &addArgc); + + if (addArgc) { + int32_t slotOffset = + slotIndex * sizeof(JS::Value) + BaselineStubFrameLayout::Size(); + BaseValueIndex slotAddr(FramePointer, argcReg, slotOffset); + masm.unboxObject(slotAddr, dest); + } else { + int32_t slotOffset = + slotIndex * sizeof(JS::Value) + BaselineStubFrameLayout::Size(); + Address slotAddr(FramePointer, slotOffset); + masm.unboxObject(slotAddr, dest); + } +} + +template +void BaselineCacheIRCompiler::storeThis(const T& newThis, Register argcReg, + CallFlags flags) { + switch (flags.getArgFormat()) { + case CallFlags::Standard: { + BaseValueIndex thisAddress( + FramePointer, + argcReg, // Arguments + 1 * sizeof(Value) + // NewTarget + BaselineStubFrameLayout::Size()); // Stub frame + masm.storeValue(newThis, thisAddress); + } break; + case CallFlags::Spread: { + Address thisAddress(FramePointer, + 2 * sizeof(Value) + // Arg array, NewTarget + BaselineStubFrameLayout::Size()); // Stub frame + masm.storeValue(newThis, thisAddress); + } break; + default: + MOZ_CRASH("Invalid arg format for scripted constructor"); + } +} + +/* + * Scripted constructors require a |this| object to be created prior to the + * call. When this function is called, the stack looks like (bottom->top): + * + * [..., Callee, ThisV, Arg0V, ..., ArgNV, NewTarget, StubFrameHeader] + * + * At this point, |ThisV| is JSWhyMagic::JS_IS_CONSTRUCTING. + * + * This function calls CreateThis to generate a new |this| object, then + * overwrites the magic ThisV on the stack. + */ +void BaselineCacheIRCompiler::createThis(Register argcReg, Register calleeReg, + Register scratch, CallFlags flags, + bool isBoundFunction) { + MOZ_ASSERT(flags.isConstructing()); + + if (flags.needsUninitializedThis()) { + storeThis(MagicValue(JS_UNINITIALIZED_LEXICAL), argcReg, flags); + return; + } + + // Save live registers that don't have to be traced. + LiveGeneralRegisterSet liveNonGCRegs; + liveNonGCRegs.add(argcReg); + liveNonGCRegs.add(ICStubReg); + masm.PushRegsInMask(liveNonGCRegs); + + // CreateThis takes two arguments: callee, and newTarget. + + if (isBoundFunction) { + // Push the bound function's target as callee and newTarget. + Address boundTarget(calleeReg, BoundFunctionObject::offsetOfTargetSlot()); + masm.unboxObject(boundTarget, scratch); + masm.push(scratch); + masm.push(scratch); + } else { + // Push newTarget: + loadStackObject(ArgumentKind::NewTarget, flags, argcReg, scratch); + masm.push(scratch); + + // Push callee: + loadStackObject(ArgumentKind::Callee, flags, argcReg, scratch); + masm.push(scratch); + } + + // Call CreateThisFromIC. + using Fn = + bool (*)(JSContext*, HandleObject, HandleObject, MutableHandleValue); + callVM(masm); + +#ifdef DEBUG + Label createdThisOK; + masm.branchTestObject(Assembler::Equal, JSReturnOperand, &createdThisOK); + masm.branchTestMagic(Assembler::Equal, JSReturnOperand, &createdThisOK); + masm.assumeUnreachable( + "The return of CreateThis must be an object or uninitialized."); + masm.bind(&createdThisOK); +#endif + + // Restore saved registers. + masm.PopRegsInMask(liveNonGCRegs); + + // Save |this| value back into pushed arguments on stack. + MOZ_ASSERT(!liveNonGCRegs.aliases(JSReturnOperand)); + storeThis(JSReturnOperand, argcReg, flags); + + // Restore calleeReg. CreateThisFromIC may trigger a GC, so we reload the + // callee from the stub frame (which is traced) instead of spilling it to + // the stack. + loadStackObject(ArgumentKind::Callee, flags, argcReg, calleeReg); +} + +void BaselineCacheIRCompiler::updateReturnValue() { + Label skipThisReplace; + masm.branchTestObject(Assembler::Equal, JSReturnOperand, &skipThisReplace); + + // If a constructor does not explicitly return an object, the return value + // of the constructor is |this|. We load it out of the baseline stub frame. + + // At this point, the stack looks like this: + // newTarget + // ArgN + // ... + // Arg0 + // ThisVal <---- We want this value. + // Callee token | Skip two stack slots. + // Frame descriptor v + // [Top of stack] + size_t thisvOffset = + JitFrameLayout::offsetOfThis() - JitFrameLayout::bytesPoppedAfterCall(); + Address thisAddress(masm.getStackPointer(), thisvOffset); + masm.loadValue(thisAddress, JSReturnOperand); + +#ifdef DEBUG + masm.branchTestObject(Assembler::Equal, JSReturnOperand, &skipThisReplace); + masm.assumeUnreachable("Return of constructing call should be an object."); +#endif + masm.bind(&skipThisReplace); +} + +bool BaselineCacheIRCompiler::emitCallScriptedFunction(ObjOperandId calleeId, + Int32OperandId argcId, + CallFlags flags, + uint32_t argcFixed) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + AutoScratchRegister scratch2(allocator, masm); + + Register calleeReg = allocator.useRegister(masm, calleeId); + Register argcReg = allocator.useRegister(masm, argcId); + + bool isConstructing = flags.isConstructing(); + bool isSameRealm = flags.isSameRealm(); + + if (!updateArgc(flags, argcReg, scratch)) { + return false; + } + + allocator.discardStack(masm); + + // Push a stub frame so that we can perform a non-tail call. + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + if (!isSameRealm) { + masm.switchToObjectRealm(calleeReg, scratch); + } + + if (isConstructing) { + createThis(argcReg, calleeReg, scratch, flags, + /* isBoundFunction = */ false); + } + + pushArguments(argcReg, calleeReg, scratch, scratch2, flags, argcFixed, + /*isJitCall =*/true); + + // Load the start of the target JitCode. + Register code = scratch2; + masm.loadJitCodeRaw(calleeReg, code); + + // Note that we use Push, not push, so that callJit will align the stack + // properly on ARM. + masm.PushCalleeToken(calleeReg, isConstructing); + masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, argcReg, scratch); + + // Handle arguments underflow. + Label noUnderflow; + masm.loadFunctionArgCount(calleeReg, calleeReg); + masm.branch32(Assembler::AboveOrEqual, argcReg, calleeReg, &noUnderflow); + { + // Call the arguments rectifier. + TrampolinePtr argumentsRectifier = + cx_->runtime()->jitRuntime()->getArgumentsRectifier(); + masm.movePtr(argumentsRectifier, code); + } + + masm.bind(&noUnderflow); + masm.callJit(code); + + // If this is a constructing call, and the callee returns a non-object, + // replace it with the |this| object passed in. + if (isConstructing) { + updateReturnValue(); + } + + stubFrame.leave(masm); + + if (!isSameRealm) { + masm.switchToBaselineFrameRealm(scratch2); + } + + return true; +} + +bool BaselineCacheIRCompiler::emitCallWasmFunction( + ObjOperandId calleeId, Int32OperandId argcId, CallFlags flags, + uint32_t argcFixed, uint32_t funcExportOffset, uint32_t instanceOffset) { + return emitCallScriptedFunction(calleeId, argcId, flags, argcFixed); +} + +bool BaselineCacheIRCompiler::emitCallInlinedFunction(ObjOperandId calleeId, + Int32OperandId argcId, + uint32_t icScriptOffset, + CallFlags flags, + uint32_t argcFixed) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + AutoScratchRegisterMaybeOutputType scratch2(allocator, masm, output); + AutoScratchRegister codeReg(allocator, masm); + + Register calleeReg = allocator.useRegister(masm, calleeId); + Register argcReg = allocator.useRegister(masm, argcId); + + bool isConstructing = flags.isConstructing(); + bool isSameRealm = flags.isSameRealm(); + + FailurePath* failure; + if (!addFailurePath(&failure)) { + return false; + } + + masm.loadBaselineJitCodeRaw(calleeReg, codeReg, failure->label()); + + if (!updateArgc(flags, argcReg, scratch)) { + return false; + } + + allocator.discardStack(masm); + + // Push a stub frame so that we can perform a non-tail call. + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + if (!isSameRealm) { + masm.switchToObjectRealm(calleeReg, scratch); + } + + Label baselineScriptDiscarded; + if (isConstructing) { + createThis(argcReg, calleeReg, scratch, flags, + /* isBoundFunction = */ false); + + // CreateThisFromIC may trigger a GC and discard the BaselineScript. + // We have already called discardStack, so we can't use a FailurePath. + // Instead, we skip storing the ICScript in the JSContext and use a + // normal non-inlined call. + masm.loadBaselineJitCodeRaw(calleeReg, codeReg, &baselineScriptDiscarded); + } + + // Store icScript in the context. + Address icScriptAddr(stubAddress(icScriptOffset)); + masm.loadPtr(icScriptAddr, scratch); + masm.storeICScriptInJSContext(scratch); + + if (isConstructing) { + Label skip; + masm.jump(&skip); + masm.bind(&baselineScriptDiscarded); + masm.loadJitCodeRaw(calleeReg, codeReg); + masm.bind(&skip); + } + + pushArguments(argcReg, calleeReg, scratch, scratch2, flags, argcFixed, + /*isJitCall =*/true); + + // Note that we use Push, not push, so that callJit will align the stack + // properly on ARM. + masm.PushCalleeToken(calleeReg, isConstructing); + masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, argcReg, scratch); + + // Handle arguments underflow. + Label noUnderflow; + masm.loadFunctionArgCount(calleeReg, calleeReg); + masm.branch32(Assembler::AboveOrEqual, argcReg, calleeReg, &noUnderflow); + + // Call the trial-inlining arguments rectifier. + ArgumentsRectifierKind kind = ArgumentsRectifierKind::TrialInlining; + TrampolinePtr argumentsRectifier = + cx_->runtime()->jitRuntime()->getArgumentsRectifier(kind); + masm.movePtr(argumentsRectifier, codeReg); + + masm.bind(&noUnderflow); + masm.callJit(codeReg); + + // If this is a constructing call, and the callee returns a non-object, + // replace it with the |this| object passed in. + if (isConstructing) { + updateReturnValue(); + } + + stubFrame.leave(masm); + + if (!isSameRealm) { + masm.switchToBaselineFrameRealm(codeReg); + } + + return true; +} + +bool BaselineCacheIRCompiler::emitCallBoundScriptedFunction( + ObjOperandId calleeId, ObjOperandId targetId, Int32OperandId argcId, + CallFlags flags, uint32_t numBoundArgs) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch(allocator, masm, output); + AutoScratchRegister scratch2(allocator, masm); + + Register calleeReg = allocator.useRegister(masm, calleeId); + Register argcReg = allocator.useRegister(masm, argcId); + + bool isConstructing = flags.isConstructing(); + bool isSameRealm = flags.isSameRealm(); + + allocator.discardStack(masm); + + // Push a stub frame so that we can perform a non-tail call. + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + Address boundTarget(calleeReg, BoundFunctionObject::offsetOfTargetSlot()); + + // If we're constructing, switch to the target's realm and create |this|. If + // we're not constructing, we switch to the target's realm after pushing the + // arguments and loading the target. + if (isConstructing) { + if (!isSameRealm) { + masm.unboxObject(boundTarget, scratch); + masm.switchToObjectRealm(scratch, scratch); + } + createThis(argcReg, calleeReg, scratch, flags, + /* isBoundFunction = */ true); + } + + // Push all arguments, including |this|. + pushBoundFunctionArguments(argcReg, calleeReg, scratch, scratch2, flags, + numBoundArgs, /* isJitCall = */ true); + + // Load the target JSFunction. + masm.unboxObject(boundTarget, calleeReg); + + if (!isConstructing && !isSameRealm) { + masm.switchToObjectRealm(calleeReg, scratch); + } + + // Update argc. + masm.add32(Imm32(numBoundArgs), argcReg); + + // Load the start of the target JitCode. + Register code = scratch2; + masm.loadJitCodeRaw(calleeReg, code); + + // Note that we use Push, not push, so that callJit will align the stack + // properly on ARM. + masm.PushCalleeToken(calleeReg, isConstructing); + masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, argcReg, scratch); + + // Handle arguments underflow. + Label noUnderflow; + masm.loadFunctionArgCount(calleeReg, calleeReg); + masm.branch32(Assembler::AboveOrEqual, argcReg, calleeReg, &noUnderflow); + { + // Call the arguments rectifier. + TrampolinePtr argumentsRectifier = + cx_->runtime()->jitRuntime()->getArgumentsRectifier(); + masm.movePtr(argumentsRectifier, code); + } + + masm.bind(&noUnderflow); + masm.callJit(code); + + if (isConstructing) { + updateReturnValue(); + } + + stubFrame.leave(masm); + + if (!isSameRealm) { + masm.switchToBaselineFrameRealm(scratch2); + } + + return true; +} + +bool BaselineCacheIRCompiler::emitNewArrayObjectResult(uint32_t arrayLength, + uint32_t shapeOffset, + uint32_t siteOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + gc::AllocKind allocKind = GuessArrayGCKind(arrayLength); + MOZ_ASSERT(CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_)); + allocKind = ForegroundToBackgroundAllocKind(allocKind); + + uint32_t slotCount = GetGCKindSlots(allocKind); + MOZ_ASSERT(slotCount >= ObjectElements::VALUES_PER_HEADER); + uint32_t arrayCapacity = slotCount - ObjectElements::VALUES_PER_HEADER; + + AutoOutputRegister output(*this); + AutoScratchRegister result(allocator, masm); + AutoScratchRegister scratch(allocator, masm); + AutoScratchRegister site(allocator, masm); + AutoScratchRegisterMaybeOutput shape(allocator, masm, output); + + Address shapeAddr(stubAddress(shapeOffset)); + masm.loadPtr(shapeAddr, shape); + + Address siteAddr(stubAddress(siteOffset)); + masm.loadPtr(siteAddr, site); + + allocator.discardStack(masm); + + Label done; + Label fail; + + masm.createArrayWithFixedElements(result, shape, scratch, arrayLength, + arrayCapacity, allocKind, gc::Heap::Default, + &fail, AllocSiteInput(site)); + masm.jump(&done); + + { + masm.bind(&fail); + + // We get here if the nursery is full (unlikely) but also for tenured + // allocations if the current arena is full and we need to allocate a new + // one (fairly common). + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.Push(site); + masm.Push(Imm32(int32_t(allocKind))); + masm.Push(Imm32(arrayLength)); + + using Fn = + ArrayObject* (*)(JSContext*, uint32_t, gc::AllocKind, gc::AllocSite*); + callVM(masm); + + stubFrame.leave(masm); + masm.storeCallPointerResult(result); + } + + masm.bind(&done); + masm.tagValue(JSVAL_TYPE_OBJECT, result, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitNewPlainObjectResult(uint32_t numFixedSlots, + uint32_t numDynamicSlots, + gc::AllocKind allocKind, + uint32_t shapeOffset, + uint32_t siteOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegister obj(allocator, masm); + AutoScratchRegister scratch(allocator, masm); + AutoScratchRegister site(allocator, masm); + AutoScratchRegisterMaybeOutput shape(allocator, masm, output); + + Address shapeAddr(stubAddress(shapeOffset)); + masm.loadPtr(shapeAddr, shape); + + Address siteAddr(stubAddress(siteOffset)); + masm.loadPtr(siteAddr, site); + + allocator.discardStack(masm); + + Label done; + Label fail; + + masm.createPlainGCObject(obj, shape, scratch, shape, numFixedSlots, + numDynamicSlots, allocKind, gc::Heap::Default, &fail, + AllocSiteInput(site)); + masm.jump(&done); + + { + masm.bind(&fail); + + // We get here if the nursery is full (unlikely) but also for tenured + // allocations if the current arena is full and we need to allocate a new + // one (fairly common). + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + masm.Push(site); + masm.Push(Imm32(int32_t(allocKind))); + masm.loadPtr(shapeAddr, shape); // This might have been overwritten. + masm.Push(shape); + + using Fn = JSObject* (*)(JSContext*, Handle, gc::AllocKind, + gc::AllocSite*); + callVM(masm); + + stubFrame.leave(masm); + masm.storeCallPointerResult(obj); + } + + masm.bind(&done); + masm.tagValue(JSVAL_TYPE_OBJECT, obj, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitBindFunctionResult( + ObjOperandId targetId, uint32_t argc, uint32_t templateObjectOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegister scratch(allocator, masm); + + Register target = allocator.useRegister(masm, targetId); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Push the arguments in reverse order. + for (uint32_t i = 0; i < argc; i++) { + Address argAddress(FramePointer, + BaselineStubFrameLayout::Size() + i * sizeof(Value)); + masm.pushValue(argAddress); + } + masm.moveStackPtrTo(scratch.get()); + + masm.Push(ImmWord(0)); // nullptr for maybeBound + masm.Push(Imm32(argc)); + masm.Push(scratch); + masm.Push(target); + + using Fn = BoundFunctionObject* (*)(JSContext*, Handle, Value*, + uint32_t, Handle); + callVM(masm); + + stubFrame.leave(masm); + masm.storeCallPointerResult(scratch); + + masm.tagValue(JSVAL_TYPE_OBJECT, scratch, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitSpecializedBindFunctionResult( + ObjOperandId targetId, uint32_t argc, uint32_t templateObjectOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + AutoScratchRegisterMaybeOutput scratch1(allocator, masm); + AutoScratchRegister scratch2(allocator, masm); + + Register target = allocator.useRegister(masm, targetId); + + StubFieldOffset objectField(templateObjectOffset, StubField::Type::JSObject); + emitLoadStubField(objectField, scratch2); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch1); + + // Push the arguments in reverse order. + for (uint32_t i = 0; i < argc; i++) { + Address argAddress(FramePointer, + BaselineStubFrameLayout::Size() + i * sizeof(Value)); + masm.pushValue(argAddress); + } + masm.moveStackPtrTo(scratch1.get()); + + masm.Push(scratch2); + masm.Push(Imm32(argc)); + masm.Push(scratch1); + masm.Push(target); + + using Fn = BoundFunctionObject* (*)(JSContext*, Handle, Value*, + uint32_t, Handle); + callVM(masm); + + stubFrame.leave(masm); + masm.storeCallPointerResult(scratch1); + + masm.tagValue(JSVAL_TYPE_OBJECT, scratch1, output.valueReg()); + return true; +} + +bool BaselineCacheIRCompiler::emitCloseIterScriptedResult( + ObjOperandId iterId, ObjOperandId calleeId, CompletionKind kind, + uint32_t calleeNargs) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + Register iter = allocator.useRegister(masm, iterId); + Register callee = allocator.useRegister(masm, calleeId); + + AutoScratchRegister code(allocator, masm); + AutoScratchRegister scratch(allocator, masm); + + masm.loadJitCodeRaw(callee, code); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + // Call the return method. + masm.alignJitStackBasedOnNArgs(calleeNargs, /*countIncludesThis = */ false); + for (uint32_t i = 0; i < calleeNargs; i++) { + masm.pushValue(UndefinedValue()); + } + masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(iter))); + masm.Push(callee); + masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, /* argc = */ 0); + + masm.callJit(code); + + if (kind != CompletionKind::Throw) { + // Verify that the return value is an object. + Label success; + masm.branchTestObject(Assembler::Equal, JSReturnOperand, &success); + + masm.Push(Imm32(int32_t(CheckIsObjectKind::IteratorReturn))); + using Fn = bool (*)(JSContext*, CheckIsObjectKind); + callVM(masm); + + masm.bind(&success); + } + + stubFrame.leave(masm); + return true; +} + +static void CallRegExpStub(MacroAssembler& masm, size_t jitRealmStubOffset, + Register temp, Label* vmCall) { + // Call cx->realm()->jitRealm()->regExpStub. We store a pointer to the RegExp + // stub in the IC stub to keep it alive, but we shouldn't use it if the stub + // has been discarded in the meantime (because we might have changed GC string + // pretenuring heuristics that affect behavior of the stub). This is uncommon + // but can happen if we discarded all JIT code but had some active (Baseline) + // scripts on the stack. + masm.loadJSContext(temp); + masm.loadPtr(Address(temp, JSContext::offsetOfRealm()), temp); + masm.loadPtr(Address(temp, Realm::offsetOfJitRealm()), temp); + masm.loadPtr(Address(temp, jitRealmStubOffset), temp); + masm.branchTestPtr(Assembler::Zero, temp, temp, vmCall); + masm.call(Address(temp, JitCode::offsetOfCode())); +} + +// Used to move inputs to the registers expected by the RegExp stub. +static void SetRegExpStubInputRegisters(MacroAssembler& masm, + Register* regexpSrc, + Register regexpDest, Register* inputSrc, + Register inputDest, + Register* lastIndexSrc, + Register lastIndexDest) { + MoveResolver& moves = masm.moveResolver(); + if (*regexpSrc != regexpDest) { + masm.propagateOOM(moves.addMove(MoveOperand(*regexpSrc), + MoveOperand(regexpDest), MoveOp::GENERAL)); + *regexpSrc = regexpDest; + } + if (*inputSrc != inputDest) { + masm.propagateOOM(moves.addMove(MoveOperand(*inputSrc), + MoveOperand(inputDest), MoveOp::GENERAL)); + *inputSrc = inputDest; + } + if (lastIndexSrc && *lastIndexSrc != lastIndexDest) { + masm.propagateOOM(moves.addMove(MoveOperand(*lastIndexSrc), + MoveOperand(lastIndexDest), MoveOp::INT32)); + *lastIndexSrc = lastIndexDest; + } + + masm.propagateOOM(moves.resolve()); + + MoveEmitter emitter(masm); + emitter.emit(moves); + emitter.finish(); +} + +bool BaselineCacheIRCompiler::emitCallRegExpMatcherResult( + ObjOperandId regexpId, StringOperandId inputId, Int32OperandId lastIndexId, + uint32_t stubOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register regexp = allocator.useRegister(masm, regexpId); + Register input = allocator.useRegister(masm, inputId); + Register lastIndex = allocator.useRegister(masm, lastIndexId); + Register scratch = output.valueReg().scratchReg(); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + SetRegExpStubInputRegisters(masm, ®exp, RegExpMatcherRegExpReg, &input, + RegExpMatcherStringReg, &lastIndex, + RegExpMatcherLastIndexReg); + + masm.reserveStack(RegExpReservedStack); + + Label done, vmCall, vmCallNoMatches; + CallRegExpStub(masm, JitRealm::offsetOfRegExpMatcherStub(), scratch, + &vmCallNoMatches); + masm.branchTestUndefined(Assembler::Equal, JSReturnOperand, &vmCall); + + masm.jump(&done); + + { + Label pushedMatches; + masm.bind(&vmCallNoMatches); + masm.push(ImmWord(0)); + masm.jump(&pushedMatches); + + masm.bind(&vmCall); + masm.computeEffectiveAddress( + Address(masm.getStackPointer(), InputOutputDataSize), scratch); + masm.Push(scratch); + + masm.bind(&pushedMatches); + masm.Push(lastIndex); + masm.Push(input); + masm.Push(regexp); + + using Fn = bool (*)(JSContext*, HandleObject regexp, HandleString input, + int32_t lastIndex, MatchPairs* pairs, + MutableHandleValue output); + callVM(masm); + } + + masm.bind(&done); + + static_assert(R0 == JSReturnOperand); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitCallRegExpSearcherResult( + ObjOperandId regexpId, StringOperandId inputId, Int32OperandId lastIndexId, + uint32_t stubOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register regexp = allocator.useRegister(masm, regexpId); + Register input = allocator.useRegister(masm, inputId); + Register lastIndex = allocator.useRegister(masm, lastIndexId); + Register scratch = output.valueReg().scratchReg(); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + SetRegExpStubInputRegisters(masm, ®exp, RegExpSearcherRegExpReg, &input, + RegExpSearcherStringReg, &lastIndex, + RegExpSearcherLastIndexReg); + // Ensure `scratch` doesn't conflict with the stub's input registers. + scratch = ReturnReg; + + masm.reserveStack(RegExpReservedStack); + + Label done, vmCall, vmCallNoMatches; + CallRegExpStub(masm, JitRealm::offsetOfRegExpSearcherStub(), scratch, + &vmCallNoMatches); + masm.branch32(Assembler::Equal, scratch, Imm32(RegExpSearcherResultFailed), + &vmCall); + + masm.jump(&done); + + { + Label pushedMatches; + masm.bind(&vmCallNoMatches); + masm.push(ImmWord(0)); + masm.jump(&pushedMatches); + + masm.bind(&vmCall); + masm.computeEffectiveAddress( + Address(masm.getStackPointer(), InputOutputDataSize), scratch); + masm.Push(scratch); + + masm.bind(&pushedMatches); + masm.Push(lastIndex); + masm.Push(input); + masm.Push(regexp); + + using Fn = bool (*)(JSContext*, HandleObject regexp, HandleString input, + int32_t lastIndex, MatchPairs* pairs, int32_t* result); + callVM(masm); + } + + masm.bind(&done); + + masm.tagValue(JSVAL_TYPE_INT32, ReturnReg, output.valueReg()); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitRegExpBuiltinExecMatchResult( + ObjOperandId regexpId, StringOperandId inputId, uint32_t stubOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register regexp = allocator.useRegister(masm, regexpId); + Register input = allocator.useRegister(masm, inputId); + Register scratch = output.valueReg().scratchReg(); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + SetRegExpStubInputRegisters(masm, ®exp, RegExpMatcherRegExpReg, &input, + RegExpMatcherStringReg, nullptr, InvalidReg); + + masm.reserveStack(RegExpReservedStack); + + Label done, vmCall, vmCallNoMatches; + CallRegExpStub(masm, JitRealm::offsetOfRegExpExecMatchStub(), scratch, + &vmCallNoMatches); + masm.branchTestUndefined(Assembler::Equal, JSReturnOperand, &vmCall); + + masm.jump(&done); + + { + Label pushedMatches; + masm.bind(&vmCallNoMatches); + masm.push(ImmWord(0)); + masm.jump(&pushedMatches); + + masm.bind(&vmCall); + masm.computeEffectiveAddress( + Address(masm.getStackPointer(), InputOutputDataSize), scratch); + masm.Push(scratch); + + masm.bind(&pushedMatches); + masm.Push(input); + masm.Push(regexp); + + using Fn = + bool (*)(JSContext*, Handle regexp, HandleString input, + MatchPairs* pairs, MutableHandleValue output); + callVM(masm); + } + + masm.bind(&done); + + static_assert(R0 == JSReturnOperand); + + stubFrame.leave(masm); + return true; +} + +bool BaselineCacheIRCompiler::emitRegExpBuiltinExecTestResult( + ObjOperandId regexpId, StringOperandId inputId, uint32_t stubOffset) { + JitSpew(JitSpew_Codegen, "%s", __FUNCTION__); + + AutoOutputRegister output(*this); + Register regexp = allocator.useRegister(masm, regexpId); + Register input = allocator.useRegister(masm, inputId); + Register scratch = output.valueReg().scratchReg(); + + allocator.discardStack(masm); + + AutoStubFrame stubFrame(*this); + stubFrame.enter(masm, scratch); + + SetRegExpStubInputRegisters(masm, ®exp, RegExpExecTestRegExpReg, &input, + RegExpExecTestStringReg, nullptr, InvalidReg); + // Ensure `scratch` doesn't conflict with the stub's input registers. + scratch = ReturnReg; + + Label done, vmCall; + CallRegExpStub(masm, JitRealm::offsetOfRegExpExecTestStub(), scratch, + &vmCall); + masm.branch32(Assembler::Equal, scratch, Imm32(RegExpExecTestResultFailed), + &vmCall); + + masm.jump(&done); + + { + masm.bind(&vmCall); + + masm.Push(input); + masm.Push(regexp); + + using Fn = bool (*)(JSContext*, Handle regexp, + HandleString input, bool* result); + callVM(masm); + } + + masm.bind(&done); + + masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, output.valueReg()); + + stubFrame.leave(masm); + return true; +} diff --git a/js/src/jit/BaselineCacheIRCompiler.h b/js/src/jit/BaselineCacheIRCompiler.h new file mode 100644 index 0000000000..91d1aa55f0 --- /dev/null +++ b/js/src/jit/BaselineCacheIRCompiler.h @@ -0,0 +1,150 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_BaselineCacheIRCompiler_h +#define jit_BaselineCacheIRCompiler_h + +#include "mozilla/Attributes.h" +#include "mozilla/Maybe.h" + +#include +#include + +#include "jstypes.h" + +#include "jit/CacheIR.h" +#include "jit/CacheIRCompiler.h" +#include "jit/CacheIROpsGenerated.h" +#include "jit/CacheIRReader.h" + +struct JS_PUBLIC_API JSContext; + +class JSScript; + +namespace js { +namespace jit { + +class CacheIRWriter; +class ICFallbackStub; +class ICScript; +class JitCode; +class Label; +class MacroAssembler; + +struct Address; +struct Register; + +enum class ICAttachResult { Attached, DuplicateStub, TooLarge, OOM }; + +bool TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback, JSScript* script, + ICScript* icScript); + +ICAttachResult AttachBaselineCacheIRStub(JSContext* cx, + const CacheIRWriter& writer, + CacheKind kind, JSScript* outerScript, + ICScript* icScript, + ICFallbackStub* stub, + const char* name); + +// BaselineCacheIRCompiler compiles CacheIR to BaselineIC native code. +class MOZ_RAII BaselineCacheIRCompiler : public CacheIRCompiler { + bool makesGCCalls_; + Register baselineFrameReg_ = FramePointer; + + // This register points to the baseline frame of the caller. It should only + // be used before we enter a stub frame. This is normally the frame pointer + // register, but with --enable-ic-frame-pointers we have to allocate a + // separate register. + inline Register baselineFrameReg() { + MOZ_ASSERT(!enteredStubFrame_); + return baselineFrameReg_; + } + + [[nodiscard]] bool emitStoreSlotShared(bool isFixed, ObjOperandId objId, + uint32_t offsetOffset, + ValOperandId rhsId); + [[nodiscard]] bool emitAddAndStoreSlotShared( + CacheOp op, ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId, + uint32_t newShapeOffset, mozilla::Maybe numNewSlotsOffset); + + bool updateArgc(CallFlags flags, Register argcReg, Register scratch); + void loadStackObject(ArgumentKind kind, CallFlags flags, Register argcReg, + Register dest); + void pushArguments(Register argcReg, Register calleeReg, Register scratch, + Register scratch2, CallFlags flags, uint32_t argcFixed, + bool isJitCall); + void pushStandardArguments(Register argcReg, Register scratch, + Register scratch2, uint32_t argcFixed, + bool isJitCall, bool isConstructing); + void pushArrayArguments(Register argcReg, Register scratch, Register scratch2, + bool isJitCall, bool isConstructing); + void pushFunCallArguments(Register argcReg, Register calleeReg, + Register scratch, Register scratch2, + uint32_t argcFixed, bool isJitCall); + void pushFunApplyArgsObj(Register argcReg, Register calleeReg, + Register scratch, Register scratch2, bool isJitCall); + void pushFunApplyNullUndefinedArguments(Register calleeReg, bool isJitCall); + void pushBoundFunctionArguments(Register argcReg, Register calleeReg, + Register scratch, Register scratch2, + CallFlags flags, uint32_t numBoundArgs, + bool isJitCall); + void createThis(Register argcReg, Register calleeReg, Register scratch, + CallFlags flags, bool isBoundFunction); + template + void storeThis(const T& newThis, Register argcReg, CallFlags flags); + void updateReturnValue(); + + enum class NativeCallType { Native, ClassHook }; + bool emitCallNativeShared(NativeCallType callType, ObjOperandId calleeId, + Int32OperandId argcId, CallFlags flags, + uint32_t argcFixed, + mozilla::Maybe ignoresReturnValue, + mozilla::Maybe targetOffset); + + enum class StringCode { CodeUnit, CodePoint }; + bool emitStringFromCodeResult(Int32OperandId codeId, StringCode stringCode); + + void emitAtomizeString(Register str, Register temp, Label* failure); + + bool emitCallScriptedGetterShared(ValOperandId receiverId, + uint32_t getterOffset, bool sameRealm, + uint32_t nargsAndFlagsOffset, + mozilla::Maybe icScriptOffset); + bool emitCallScriptedSetterShared(ObjOperandId receiverId, + uint32_t setterOffset, ValOperandId rhsId, + bool sameRealm, + uint32_t nargsAndFlagsOffset, + mozilla::Maybe icScriptOffset); + + BaselineICPerfSpewer perfSpewer_; + + public: + BaselineICPerfSpewer& perfSpewer() { return perfSpewer_; } + + friend class AutoStubFrame; + + BaselineCacheIRCompiler(JSContext* cx, TempAllocator& alloc, + const CacheIRWriter& writer, uint32_t stubDataOffset); + + [[nodiscard]] bool init(CacheKind kind); + + template + void callVM(MacroAssembler& masm); + + JitCode* compile(); + + bool makesGCCalls() const; + + Address stubAddress(uint32_t offset) const; + + private: + CACHE_IR_COMPILER_UNSHARED_GENERATED +}; + +} // namespace jit +} // namespace js + +#endif /* jit_BaselineCacheIRCompiler_h */ diff --git a/js/src/jit/BaselineCodeGen.cpp b/js/src/jit/BaselineCodeGen.cpp new file mode 100644 index 0000000000..f88a026074 --- /dev/null +++ b/js/src/jit/BaselineCodeGen.cpp @@ -0,0 +1,6897 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jit/BaselineCodeGen.h" + +#include "mozilla/Casting.h" + +#include "gc/GC.h" +#include "jit/BaselineIC.h" +#include "jit/BaselineJIT.h" +#include "jit/CacheIRCompiler.h" +#include "jit/CacheIRGenerator.h" +#include "jit/CalleeToken.h" +#include "jit/FixedList.h" +#include "jit/IonOptimizationLevels.h" +#include "jit/JitcodeMap.h" +#include "jit/JitFrames.h" +#include "jit/JitRuntime.h" +#include "jit/JitSpewer.h" +#include "jit/Linker.h" +#include "jit/PerfSpewer.h" +#include "jit/SharedICHelpers.h" +#include "jit/TemplateObject.h" +#include "jit/TrialInlining.h" +#include "jit/VMFunctions.h" +#include "js/friend/ErrorMessages.h" // JSMSG_* +#include "js/UniquePtr.h" +#include "vm/AsyncFunction.h" +#include "vm/AsyncIteration.h" +#include "vm/BuiltinObjectKind.h" +#include "vm/EnvironmentObject.h" +#include "vm/FunctionFlags.h" // js::FunctionFlags +#include "vm/Interpreter.h" +#include "vm/JSFunction.h" +#include "vm/Time.h" +#ifdef MOZ_VTUNE +# include "vtune/VTuneWrapper.h" +#endif + +#include "debugger/DebugAPI-inl.h" +#include "jit/BaselineFrameInfo-inl.h" +#include "jit/JitHints-inl.h" +#include "jit/JitScript-inl.h" +#include "jit/MacroAssembler-inl.h" +#include "jit/SharedICHelpers-inl.h" +#include "jit/TemplateObject-inl.h" +#include "jit/VMFunctionList-inl.h" +#include "vm/Interpreter-inl.h" +#include "vm/JSScript-inl.h" + +using namespace js; +using namespace js::jit; + +using JS::TraceKind; + +using mozilla::AssertedCast; +using mozilla::Maybe; + +namespace js { + +class PlainObject; + +namespace jit { + +BaselineCompilerHandler::BaselineCompilerHandler(JSContext* cx, + MacroAssembler& masm, + TempAllocator& alloc, + JSScript* script) + : frame_(script, masm), + alloc_(alloc), + analysis_(alloc, script), +#ifdef DEBUG + masm_(masm), +#endif + script_(script), + pc_(script->code()), + icEntryIndex_(0), + compileDebugInstrumentation_(script->isDebuggee()), + ionCompileable_(IsIonEnabled(cx) && CanIonCompileScript(cx, script)) { +} + +BaselineInterpreterHandler::BaselineInterpreterHandler(JSContext* cx, + MacroAssembler& masm) + : frame_(masm) {} + +template +template +BaselineCodeGen::BaselineCodeGen(JSContext* cx, TempAllocator& alloc, + HandlerArgs&&... args) + : handler(cx, masm, std::forward(args)...), + cx(cx), + masm(cx, alloc), + frame(handler.frame()) {} + +BaselineCompiler::BaselineCompiler(JSContext* cx, TempAllocator& alloc, + JSScript* script) + : BaselineCodeGen(cx, alloc, /* HandlerArgs = */ alloc, script), + profilerPushToggleOffset_() { +#ifdef JS_CODEGEN_NONE + MOZ_CRASH(); +#endif +} + +BaselineInterpreterGenerator::BaselineInterpreterGenerator(JSContext* cx, + TempAllocator& alloc) + : BaselineCodeGen(cx, alloc /* no handlerArgs */) {} + +bool BaselineCompilerHandler::init(JSContext* cx) { + if (!analysis_.init(alloc_)) { + return false; + } + + uint32_t len = script_->length(); + + if (!labels_.init(alloc_, len)) { + return false; + } + + for (size_t i = 0; i < len; i++) { + new (&labels_[i]) Label(); + } + + if (!frame_.init(alloc_)) { + return false; + } + + return true; +} + +bool BaselineCompiler::init() { + if (!handler.init(cx)) { + return false; + } + + return true; +} + +bool BaselineCompilerHandler::recordCallRetAddr(JSContext* cx, + RetAddrEntry::Kind kind, + uint32_t retOffset) { + uint32_t pcOffset = script_->pcToOffset(pc_); + + // Entries must be sorted by pcOffset for binary search to work. + // See BaselineScript::retAddrEntryFromPCOffset. + MOZ_ASSERT_IF(!retAddrEntries_.empty(), + retAddrEntries_.back().pcOffset() <= pcOffset); + + // Similarly, entries must be sorted by return offset and this offset must be + // unique. See BaselineScript::retAddrEntryFromReturnOffset. + MOZ_ASSERT_IF(!retAddrEntries_.empty() && !masm_.oom(), + retAddrEntries_.back().returnOffset().offset() < retOffset); + + if (!retAddrEntries_.emplaceBack(pcOffset, kind, CodeOffset(retOffset))) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +bool BaselineInterpreterHandler::recordCallRetAddr(JSContext* cx, + RetAddrEntry::Kind kind, + uint32_t retOffset) { + switch (kind) { + case RetAddrEntry::Kind::DebugPrologue: + MOZ_ASSERT(callVMOffsets_.debugPrologueOffset == 0, + "expected single DebugPrologue call"); + callVMOffsets_.debugPrologueOffset = retOffset; + break; + case RetAddrEntry::Kind::DebugEpilogue: + MOZ_ASSERT(callVMOffsets_.debugEpilogueOffset == 0, + "expected single DebugEpilogue call"); + callVMOffsets_.debugEpilogueOffset = retOffset; + break; + case RetAddrEntry::Kind::DebugAfterYield: + MOZ_ASSERT(callVMOffsets_.debugAfterYieldOffset == 0, + "expected single DebugAfterYield call"); + callVMOffsets_.debugAfterYieldOffset = retOffset; + break; + default: + break; + } + + return true; +} + +bool BaselineInterpreterHandler::addDebugInstrumentationOffset( + JSContext* cx, CodeOffset offset) { + if (!debugInstrumentationOffsets_.append(offset.offset())) { + ReportOutOfMemory(cx); + return false; + } + return true; +} + +MethodStatus BaselineCompiler::compile() { + AutoCreatedBy acb(masm, "BaselineCompiler::compile"); + + Rooted script(cx, handler.script()); + JitSpew(JitSpew_BaselineScripts, "Baseline compiling script %s:%u:%u (%p)", + script->filename(), script->lineno(), script->column(), script.get()); + + JitSpew(JitSpew_Codegen, "# Emitting baseline code for script %s:%u:%u", + script->filename(), script->lineno(), script->column()); + + AutoIncrementalTimer timer(cx->realm()->timers.baselineCompileTime); + + AutoKeepJitScripts keepJitScript(cx); + if (!script->ensureHasJitScript(cx, keepJitScript)) { + return Method_Error; + } + + // When code coverage is enabled, we have to create the ScriptCounts if they + // do not exist. + if (!script->hasScriptCounts() && cx->realm()->collectCoverageForDebug()) { + if (!script->initScriptCounts(cx)) { + return Method_Error; + } + } + + if (!JitOptions.disableJitHints && + cx->runtime()->jitRuntime()->hasJitHintsMap()) { + JitHintsMap* jitHints = cx->runtime()->jitRuntime()->getJitHintsMap(); + jitHints->setEagerBaselineHint(script); + } + + // Suppress GC during compilation. + gc::AutoSuppressGC suppressGC(cx); + + if (!script->jitScript()->ensureHasCachedBaselineJitData(cx, script)) { + return Method_Error; + } + + MOZ_ASSERT(!script->hasBaselineScript()); + + perfSpewer_.recordOffset(masm, "Prologue"); + if (!emitPrologue()) { + return Method_Error; + } + + MethodStatus status = emitBody(); + if (status != Method_Compiled) { + return status; + } + + perfSpewer_.recordOffset(masm, "Epilogue"); + if (!emitEpilogue()) { + return Method_Error; + } + + perfSpewer_.recordOffset(masm, "OOLPostBarrierSlot"); + if (!emitOutOfLinePostBarrierSlot()) { + return Method_Error; + } + + AutoCreatedBy acb2(masm, "exception_tail"); + Linker linker(masm); + if (masm.oom()) { + ReportOutOfMemory(cx); + return Method_Error; + } + + JitCode* code = linker.newCode(cx, CodeKind::Baseline); + if (!code) { + return Method_Error; + } + + UniquePtr baselineScript( + BaselineScript::New( + cx, warmUpCheckPrologueOffset_.offset(), + profilerEnterFrameToggleOffset_.offset(), + profilerExitFrameToggleOffset_.offset(), + handler.retAddrEntries().length(), handler.osrEntries().length(), + debugTrapEntries_.length(), script->resumeOffsets().size()), + JS::DeletePolicy(cx->runtime())); + if (!baselineScript) { + return Method_Error; + } + + baselineScript->setMethod(code); + + JitSpew(JitSpew_BaselineScripts, + "Created BaselineScript %p (raw %p) for %s:%u:%u", + (void*)baselineScript.get(), (void*)code->raw(), script->filename(), + script->lineno(), script->column()); + + baselineScript->copyRetAddrEntries(handler.retAddrEntries().begin()); + baselineScript->copyOSREntries(handler.osrEntries().begin()); + baselineScript->copyDebugTrapEntries(debugTrapEntries_.begin()); + + // If profiler instrumentation is enabled, toggle instrumentation on. + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled( + cx->runtime())) { + baselineScript->toggleProfilerInstrumentation(true); + } + + // Compute native resume addresses for the script's resume offsets. + baselineScript->computeResumeNativeOffsets(script, resumeOffsetEntries_); + + if (compileDebugInstrumentation()) { + baselineScript->setHasDebugInstrumentation(); + } + + // Always register a native => bytecode mapping entry, since profiler can be + // turned on with baseline jitcode on stack, and baseline jitcode cannot be + // invalidated. + { + JitSpew(JitSpew_Profiling, + "Added JitcodeGlobalEntry for baseline script %s:%u:%u (%p)", + script->filename(), script->lineno(), script->column(), + baselineScript.get()); + + // Generate profiling string. + UniqueChars str = GeckoProfilerRuntime::allocProfileString(cx, script); + if (!str) { + return Method_Error; + } + + auto entry = MakeJitcodeGlobalEntry( + cx, code, code->raw(), code->rawEnd(), script, std::move(str)); + if (!entry) { + return Method_Error; + } + + JitcodeGlobalTable* globalTable = + cx->runtime()->jitRuntime()->getJitcodeGlobalTable(); + if (!globalTable->addEntry(std::move(entry))) { + ReportOutOfMemory(cx); + return Method_Error; + } + + // Mark the jitcode as having a bytecode map. + code->setHasBytecodeMap(); + } + + script->jitScript()->setBaselineScript(script, baselineScript.release()); + + perfSpewer_.saveProfile(cx, script, code); + +#ifdef MOZ_VTUNE + vtune::MarkScript(code, script, "baseline"); +#endif + + return Method_Compiled; +} + +// On most platforms we use a dedicated bytecode PC register to avoid many +// dependent loads and stores for sequences of simple bytecode ops. This +// register must be saved/restored around VM and IC calls. +// +// On 32-bit x86 we don't have enough registers for this (because R0-R2 require +// 6 registers) so there we always store the pc on the frame. +static constexpr bool HasInterpreterPCReg() { + return InterpreterPCReg != InvalidReg; +} + +static Register LoadBytecodePC(MacroAssembler& masm, Register scratch) { + if (HasInterpreterPCReg()) { + return InterpreterPCReg; + } + + Address pcAddr(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC()); + masm.loadPtr(pcAddr, scratch); + return scratch; +} + +static void LoadInt8Operand(MacroAssembler& masm, Register dest) { + Register pc = LoadBytecodePC(masm, dest); + masm.load8SignExtend(Address(pc, sizeof(jsbytecode)), dest); +} + +static void LoadUint8Operand(MacroAssembler& masm, Register dest) { + Register pc = LoadBytecodePC(masm, dest); + masm.load8ZeroExtend(Address(pc, sizeof(jsbytecode)), dest); +} + +static void LoadUint16Operand(MacroAssembler& masm, Register dest) { + Register pc = LoadBytecodePC(masm, dest); + masm.load16ZeroExtend(Address(pc, sizeof(jsbytecode)), dest); +} + +static void LoadInt32Operand(MacroAssembler& masm, Register dest) { + Register pc = LoadBytecodePC(masm, dest); + masm.load32(Address(pc, sizeof(jsbytecode)), dest); +} + +static void LoadInt32OperandSignExtendToPtr(MacroAssembler& masm, Register pc, + Register dest) { + masm.load32SignExtendToPtr(Address(pc, sizeof(jsbytecode)), dest); +} + +static void LoadUint24Operand(MacroAssembler& masm, size_t offset, + Register dest) { + // Load the opcode and operand, then left shift to discard the opcode. + Register pc = LoadBytecodePC(masm, dest); + masm.load32(Address(pc, offset), dest); + masm.rshift32(Imm32(8), dest); +} + +static void LoadInlineValueOperand(MacroAssembler& masm, ValueOperand dest) { + // Note: the Value might be unaligned but as above we rely on all our + // platforms having appropriate support for unaligned accesses (except for + // floating point instructions on ARM). + Register pc = LoadBytecodePC(masm, dest.scratchReg()); + masm.loadUnalignedValue(Address(pc, sizeof(jsbytecode)), dest); +} + +template <> +void BaselineCompilerCodeGen::loadScript(Register dest) { + masm.movePtr(ImmGCPtr(handler.script()), dest); +} + +template <> +void BaselineInterpreterCodeGen::loadScript(Register dest) { + masm.loadPtr(frame.addressOfInterpreterScript(), dest); +} + +template <> +void BaselineCompilerCodeGen::saveInterpreterPCReg() {} + +template <> +void BaselineInterpreterCodeGen::saveInterpreterPCReg() { + if (HasInterpreterPCReg()) { + masm.storePtr(InterpreterPCReg, frame.addressOfInterpreterPC()); + } +} + +template <> +void BaselineCompilerCodeGen::restoreInterpreterPCReg() {} + +template <> +void BaselineInterpreterCodeGen::restoreInterpreterPCReg() { + if (HasInterpreterPCReg()) { + masm.loadPtr(frame.addressOfInterpreterPC(), InterpreterPCReg); + } +} + +template <> +void BaselineCompilerCodeGen::emitInitializeLocals() { + // Initialize all locals to |undefined|. Lexical bindings are temporal + // dead zoned in bytecode. + + size_t n = frame.nlocals(); + if (n == 0) { + return; + } + + // Use R0 to minimize code size. If the number of locals to push is < + // LOOP_UNROLL_FACTOR, then the initialization pushes are emitted directly + // and inline. Otherwise, they're emitted in a partially unrolled loop. + static const size_t LOOP_UNROLL_FACTOR = 4; + size_t toPushExtra = n % LOOP_UNROLL_FACTOR; + + masm.moveValue(UndefinedValue(), R0); + + // Handle any extra pushes left over by the optional unrolled loop below. + for (size_t i = 0; i < toPushExtra; i++) { + masm.pushValue(R0); + } + + // Partially unrolled loop of pushes. + if (n >= LOOP_UNROLL_FACTOR) { + size_t toPush = n - toPushExtra; + MOZ_ASSERT(toPush % LOOP_UNROLL_FACTOR == 0); + MOZ_ASSERT(toPush >= LOOP_UNROLL_FACTOR); + masm.move32(Imm32(toPush), R1.scratchReg()); + // Emit unrolled loop with 4 pushes per iteration. + Label pushLoop; + masm.bind(&pushLoop); + for (size_t i = 0; i < LOOP_UNROLL_FACTOR; i++) { + masm.pushValue(R0); + } + masm.branchSub32(Assembler::NonZero, Imm32(LOOP_UNROLL_FACTOR), + R1.scratchReg(), &pushLoop); + } +} + +template <> +void BaselineInterpreterCodeGen::emitInitializeLocals() { + // Push |undefined| for all locals. + + Register scratch = R0.scratchReg(); + loadScript(scratch); + masm.loadPtr(Address(scratch, JSScript::offsetOfSharedData()), scratch); + masm.loadPtr(Address(scratch, SharedImmutableScriptData::offsetOfISD()), + scratch); + masm.load32(Address(scratch, ImmutableScriptData::offsetOfNfixed()), scratch); + + Label top, done; + masm.branchTest32(Assembler::Zero, scratch, scratch, &done); + masm.bind(&top); + { + masm.pushValue(UndefinedValue()); + masm.branchSub32(Assembler::NonZero, Imm32(1), scratch, &top); + } + masm.bind(&done); +} + +// On input: +// R2.scratchReg() contains object being written to. +// Called with the baseline stack synced, except for R0 which is preserved. +// All other registers are usable as scratch. +// This calls: +// void PostWriteBarrier(JSRuntime* rt, JSObject* obj); +template +bool BaselineCodeGen::emitOutOfLinePostBarrierSlot() { + AutoCreatedBy acb(masm, + "BaselineCodeGen::emitOutOfLinePostBarrierSlot"); + + if (!postBarrierSlot_.used()) { + return true; + } + + masm.bind(&postBarrierSlot_); + + saveInterpreterPCReg(); + + Register objReg = R2.scratchReg(); + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + MOZ_ASSERT(!regs.has(FramePointer)); + regs.take(R0); + regs.take(objReg); + Register scratch = regs.takeAny(); +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) + // On ARM, save the link register before calling. It contains the return + // address. The |masm.ret()| later will pop this into |pc| to return. + masm.push(lr); +#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) + masm.push(ra); +#elif defined(JS_CODEGEN_LOONG64) + masm.push(ra); +#elif defined(JS_CODEGEN_RISCV64) + masm.push(ra); +#endif + masm.pushValue(R0); + + using Fn = void (*)(JSRuntime* rt, js::gc::Cell* cell); + masm.setupUnalignedABICall(scratch); + masm.movePtr(ImmPtr(cx->runtime()), scratch); + masm.passABIArg(scratch); + masm.passABIArg(objReg); + masm.callWithABI(); + + restoreInterpreterPCReg(); + + masm.popValue(R0); + masm.ret(); + return true; +} + +// Scan the a cache IR stub's fields and create an allocation site for any that +// refer to the catch-all unknown allocation site. This will be the case for +// stubs created when running in the interpreter. This happens on transition to +// baseline. +static bool CreateAllocSitesForCacheIRStub(JSScript* script, + ICCacheIRStub* stub) { + const CacheIRStubInfo* stubInfo = stub->stubInfo(); + uint8_t* stubData = stub->stubDataStart(); + + uint32_t field = 0; + size_t offset = 0; + while (true) { + StubField::Type fieldType = stubInfo->fieldType(field); + if (fieldType == StubField::Type::Limit) { + break; + } + + if (fieldType == StubField::Type::AllocSite) { + gc::AllocSite* site = + stubInfo->getPtrStubField(stub, offset); + if (site->kind() == gc::AllocSite::Kind::Unknown) { + gc::AllocSite* newSite = script->createAllocSite(); + if (!newSite) { + return false; + } + + stubInfo->replaceStubRawWord(stubData, offset, uintptr_t(site), + uintptr_t(newSite)); + } + } + + field++; + offset += StubField::sizeInBytes(fieldType); + } + + return true; +} + +static void CreateAllocSitesForICChain(JSScript* script, uint32_t entryIndex) { + JitScript* jitScript = script->jitScript(); + ICStub* stub = jitScript->icEntry(entryIndex).firstStub(); + + while (!stub->isFallback()) { + if (!CreateAllocSitesForCacheIRStub(script, stub->toCacheIRStub())) { + // This is an optimization and safe to skip if we hit OOM or per-zone + // limit. + return; + } + stub = stub->toCacheIRStub()->next(); + } +} + +template <> +bool BaselineCompilerCodeGen::emitNextIC() { + AutoCreatedBy acb(masm, "emitNextIC"); + + // Emit a call to an IC stored in JitScript. Calls to this must match the + // ICEntry order in JitScript: first the non-op IC entries for |this| and + // formal arguments, then the for-op IC entries for JOF_IC ops. + + JSScript* script = handler.script(); + uint32_t pcOffset = script->pcToOffset(handler.pc()); + + // We don't use every ICEntry and we can skip unreachable ops, so we have + // to loop until we find an ICEntry for the current pc. + const ICFallbackStub* stub; + uint32_t entryIndex; + do { + stub = script->jitScript()->fallbackStub(handler.icEntryIndex()); + entryIndex = handler.icEntryIndex(); + handler.moveToNextICEntry(); + } while (stub->pcOffset() < pcOffset); + + MOZ_ASSERT(stub->pcOffset() == pcOffset); + MOZ_ASSERT(BytecodeOpHasIC(JSOp(*handler.pc()))); + + if (BytecodeOpCanHaveAllocSite(JSOp(*handler.pc()))) { + CreateAllocSitesForICChain(script, entryIndex); + } + + // Load stub pointer into ICStubReg. + masm.loadPtr(frame.addressOfICScript(), ICStubReg); + size_t firstStubOffset = ICScript::offsetOfFirstStub(entryIndex); + masm.loadPtr(Address(ICStubReg, firstStubOffset), ICStubReg); + + CodeOffset returnOffset; + EmitCallIC(masm, &returnOffset); + + RetAddrEntry::Kind kind = RetAddrEntry::Kind::IC; + if (!handler.retAddrEntries().emplaceBack(pcOffset, kind, returnOffset)) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emitNextIC() { + saveInterpreterPCReg(); + masm.loadPtr(frame.addressOfInterpreterICEntry(), ICStubReg); + masm.loadPtr(Address(ICStubReg, ICEntry::offsetOfFirstStub()), ICStubReg); + masm.call(Address(ICStubReg, ICStub::offsetOfStubCode())); + uint32_t returnOffset = masm.currentOffset(); + restoreInterpreterPCReg(); + + // If this is an IC for a bytecode op where Ion may inline scripts, we need to + // record the return offset for Ion bailouts. + if (handler.currentOp()) { + JSOp op = *handler.currentOp(); + MOZ_ASSERT(BytecodeOpHasIC(op)); + if (IsIonInlinableOp(op)) { + if (!handler.icReturnOffsets().emplaceBack(returnOffset, op)) { + return false; + } + } + } + + return true; +} + +template <> +void BaselineCompilerCodeGen::computeFrameSize(Register dest) { + MOZ_ASSERT(!inCall_, "must not be called in the middle of a VM call"); + masm.move32(Imm32(frame.frameSize()), dest); +} + +template <> +void BaselineInterpreterCodeGen::computeFrameSize(Register dest) { + // dest := FramePointer - StackPointer. + MOZ_ASSERT(!inCall_, "must not be called in the middle of a VM call"); + masm.mov(FramePointer, dest); + masm.subStackPtrFrom(dest); +} + +template +void BaselineCodeGen::prepareVMCall() { + pushedBeforeCall_ = masm.framePushed(); +#ifdef DEBUG + inCall_ = true; +#endif + + // Ensure everything is synced. + frame.syncStack(0); +} + +template <> +void BaselineCompilerCodeGen::storeFrameSizeAndPushDescriptor( + uint32_t argSize, Register scratch) { +#ifdef DEBUG + masm.store32(Imm32(frame.frameSize()), frame.addressOfDebugFrameSize()); +#endif + + masm.pushFrameDescriptor(FrameType::BaselineJS); +} + +template <> +void BaselineInterpreterCodeGen::storeFrameSizeAndPushDescriptor( + uint32_t argSize, Register scratch) { +#ifdef DEBUG + // Store the frame size without VMFunction arguments in debug builds. + // scratch := FramePointer - StackPointer - argSize. + masm.mov(FramePointer, scratch); + masm.subStackPtrFrom(scratch); + masm.sub32(Imm32(argSize), scratch); + masm.store32(scratch, frame.addressOfDebugFrameSize()); +#endif + + masm.pushFrameDescriptor(FrameType::BaselineJS); +} + +static uint32_t GetVMFunctionArgSize(const VMFunctionData& fun) { + return fun.explicitStackSlots() * sizeof(void*); +} + +template +bool BaselineCodeGen::callVMInternal(VMFunctionId id, + RetAddrEntry::Kind kind, + CallVMPhase phase) { +#ifdef DEBUG + // Assert prepareVMCall() has been called. + MOZ_ASSERT(inCall_); + inCall_ = false; +#endif + + TrampolinePtr code = cx->runtime()->jitRuntime()->getVMWrapper(id); + const VMFunctionData& fun = GetVMFunction(id); + + uint32_t argSize = GetVMFunctionArgSize(fun); + + // Assert all arguments were pushed. + MOZ_ASSERT(masm.framePushed() - pushedBeforeCall_ == argSize); + + saveInterpreterPCReg(); + + if (phase == CallVMPhase::AfterPushingLocals) { + storeFrameSizeAndPushDescriptor(argSize, R0.scratchReg()); + } else { + MOZ_ASSERT(phase == CallVMPhase::BeforePushingLocals); +#ifdef DEBUG + uint32_t frameBaseSize = BaselineFrame::frameSizeForNumValueSlots(0); + masm.store32(Imm32(frameBaseSize), frame.addressOfDebugFrameSize()); +#endif + masm.pushFrameDescriptor(FrameType::BaselineJS); + } + MOZ_ASSERT(fun.expectTailCall == NonTailCall); + // Perform the call. + masm.call(code); + uint32_t callOffset = masm.currentOffset(); + + // Pop arguments from framePushed. + masm.implicitPop(argSize); + + restoreInterpreterPCReg(); + + return handler.recordCallRetAddr(cx, kind, callOffset); +} + +template +template +bool BaselineCodeGen::callVM(RetAddrEntry::Kind kind, + CallVMPhase phase) { + VMFunctionId fnId = VMFunctionToId::id; + return callVMInternal(fnId, kind, phase); +} + +template +bool BaselineCodeGen::emitStackCheck() { + Label skipCall; + if (handler.mustIncludeSlotsInStackCheck()) { + // Subtract the size of script->nslots() first. + Register scratch = R1.scratchReg(); + masm.moveStackPtrTo(scratch); + subtractScriptSlotsSize(scratch, R2.scratchReg()); + masm.branchPtr(Assembler::BelowOrEqual, + AbsoluteAddress(cx->addressOfJitStackLimit()), scratch, + &skipCall); + } else { + masm.branchStackPtrRhs(Assembler::BelowOrEqual, + AbsoluteAddress(cx->addressOfJitStackLimit()), + &skipCall); + } + + prepareVMCall(); + masm.loadBaselineFramePtr(FramePointer, R1.scratchReg()); + pushArg(R1.scratchReg()); + + const CallVMPhase phase = CallVMPhase::BeforePushingLocals; + const RetAddrEntry::Kind kind = RetAddrEntry::Kind::StackCheck; + + using Fn = bool (*)(JSContext*, BaselineFrame*); + if (!callVM(kind, phase)) { + return false; + } + + masm.bind(&skipCall); + return true; +} + +static void EmitCallFrameIsDebuggeeCheck(MacroAssembler& masm) { + using Fn = void (*)(BaselineFrame* frame); + masm.setupUnalignedABICall(R0.scratchReg()); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + masm.passABIArg(R0.scratchReg()); + masm.callWithABI(); +} + +template <> +bool BaselineCompilerCodeGen::emitIsDebuggeeCheck() { + if (handler.compileDebugInstrumentation()) { + EmitCallFrameIsDebuggeeCheck(masm); + } + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emitIsDebuggeeCheck() { + // Use a toggled jump to call FrameIsDebuggeeCheck only if the debugger is + // enabled. + // + // TODO(bug 1522394): consider having a cx->realm->isDebuggee guard before the + // call. Consider moving the callWithABI out-of-line. + + Label skipCheck; + CodeOffset toggleOffset = masm.toggledJump(&skipCheck); + { + saveInterpreterPCReg(); + EmitCallFrameIsDebuggeeCheck(masm); + restoreInterpreterPCReg(); + } + masm.bind(&skipCheck); + return handler.addDebugInstrumentationOffset(cx, toggleOffset); +} + +static void MaybeIncrementCodeCoverageCounter(MacroAssembler& masm, + JSScript* script, + jsbytecode* pc) { + if (!script->hasScriptCounts()) { + return; + } + PCCounts* counts = script->maybeGetPCCounts(pc); + uint64_t* counterAddr = &counts->numExec(); + masm.inc64(AbsoluteAddress(counterAddr)); +} + +template <> +bool BaselineCompilerCodeGen::emitHandleCodeCoverageAtPrologue() { + // If the main instruction is not a jump target, then we emit the + // corresponding code coverage counter. + JSScript* script = handler.script(); + jsbytecode* main = script->main(); + if (!BytecodeIsJumpTarget(JSOp(*main))) { + MaybeIncrementCodeCoverageCounter(masm, script, main); + } + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emitHandleCodeCoverageAtPrologue() { + Label skipCoverage; + CodeOffset toggleOffset = masm.toggledJump(&skipCoverage); + masm.call(handler.codeCoverageAtPrologueLabel()); + masm.bind(&skipCoverage); + return handler.codeCoverageOffsets().append(toggleOffset.offset()); +} + +template <> +void BaselineCompilerCodeGen::subtractScriptSlotsSize(Register reg, + Register scratch) { + uint32_t slotsSize = handler.script()->nslots() * sizeof(Value); + masm.subPtr(Imm32(slotsSize), reg); +} + +template <> +void BaselineInterpreterCodeGen::subtractScriptSlotsSize(Register reg, + Register scratch) { + // reg = reg - script->nslots() * sizeof(Value) + MOZ_ASSERT(reg != scratch); + loadScript(scratch); + masm.loadPtr(Address(scratch, JSScript::offsetOfSharedData()), scratch); + masm.loadPtr(Address(scratch, SharedImmutableScriptData::offsetOfISD()), + scratch); + masm.load32(Address(scratch, ImmutableScriptData::offsetOfNslots()), scratch); + static_assert(sizeof(Value) == 8, + "shift by 3 below assumes Value is 8 bytes"); + masm.lshiftPtr(Imm32(3), scratch); + masm.subPtr(scratch, reg); +} + +template <> +void BaselineCompilerCodeGen::loadGlobalLexicalEnvironment(Register dest) { + MOZ_ASSERT(!handler.script()->hasNonSyntacticScope()); + masm.movePtr(ImmGCPtr(&cx->global()->lexicalEnvironment()), dest); +} + +template <> +void BaselineInterpreterCodeGen::loadGlobalLexicalEnvironment(Register dest) { + masm.loadPtr(AbsoluteAddress(cx->addressOfRealm()), dest); + masm.loadPtr(Address(dest, Realm::offsetOfActiveGlobal()), dest); + masm.loadPrivate(Address(dest, GlobalObject::offsetOfGlobalDataSlot()), dest); + masm.loadPtr(Address(dest, GlobalObjectData::offsetOfLexicalEnvironment()), + dest); +} + +template <> +void BaselineCompilerCodeGen::pushGlobalLexicalEnvironmentValue( + ValueOperand scratch) { + frame.push(ObjectValue(cx->global()->lexicalEnvironment())); +} + +template <> +void BaselineInterpreterCodeGen::pushGlobalLexicalEnvironmentValue( + ValueOperand scratch) { + loadGlobalLexicalEnvironment(scratch.scratchReg()); + masm.tagValue(JSVAL_TYPE_OBJECT, scratch.scratchReg(), scratch); + frame.push(scratch); +} + +template <> +void BaselineCompilerCodeGen::loadGlobalThisValue(ValueOperand dest) { + JSObject* thisObj = cx->global()->lexicalEnvironment().thisObject(); + masm.moveValue(ObjectValue(*thisObj), dest); +} + +template <> +void BaselineInterpreterCodeGen::loadGlobalThisValue(ValueOperand dest) { + Register scratch = dest.scratchReg(); + loadGlobalLexicalEnvironment(scratch); + static constexpr size_t SlotOffset = + GlobalLexicalEnvironmentObject::offsetOfThisValueSlot(); + masm.loadValue(Address(scratch, SlotOffset), dest); +} + +template <> +void BaselineCompilerCodeGen::pushScriptArg() { + pushArg(ImmGCPtr(handler.script())); +} + +template <> +void BaselineInterpreterCodeGen::pushScriptArg() { + pushArg(frame.addressOfInterpreterScript()); +} + +template <> +void BaselineCompilerCodeGen::pushBytecodePCArg() { + pushArg(ImmPtr(handler.pc())); +} + +template <> +void BaselineInterpreterCodeGen::pushBytecodePCArg() { + if (HasInterpreterPCReg()) { + pushArg(InterpreterPCReg); + } else { + pushArg(frame.addressOfInterpreterPC()); + } +} + +static gc::Cell* GetScriptGCThing(JSScript* script, jsbytecode* pc, + ScriptGCThingType type) { + switch (type) { + case ScriptGCThingType::Atom: + return script->getAtom(pc); + case ScriptGCThingType::String: + return script->getString(pc); + case ScriptGCThingType::RegExp: + return script->getRegExp(pc); + case ScriptGCThingType::Object: + return script->getObject(pc); + case ScriptGCThingType::Function: + return script->getFunction(pc); + case ScriptGCThingType::Scope: + return script->getScope(pc); + case ScriptGCThingType::BigInt: + return script->getBigInt(pc); + } + MOZ_CRASH("Unexpected GCThing type"); +} + +template <> +void BaselineCompilerCodeGen::loadScriptGCThing(ScriptGCThingType type, + Register dest, + Register scratch) { + gc::Cell* thing = GetScriptGCThing(handler.script(), handler.pc(), type); + masm.movePtr(ImmGCPtr(thing), dest); +} + +template <> +void BaselineInterpreterCodeGen::loadScriptGCThing(ScriptGCThingType type, + Register dest, + Register scratch) { + MOZ_ASSERT(dest != scratch); + + // Load the index in |scratch|. + LoadInt32Operand(masm, scratch); + + // Load the GCCellPtr. + loadScript(dest); + masm.loadPtr(Address(dest, JSScript::offsetOfPrivateData()), dest); + masm.loadPtr(BaseIndex(dest, scratch, ScalePointer, + PrivateScriptData::offsetOfGCThings()), + dest); + + // Clear the tag bits. + switch (type) { + case ScriptGCThingType::Atom: + case ScriptGCThingType::String: + // Use xorPtr with a 32-bit immediate because it's more efficient than + // andPtr on 64-bit. + static_assert(uintptr_t(TraceKind::String) == 2, + "Unexpected tag bits for string GCCellPtr"); + masm.xorPtr(Imm32(2), dest); + break; + case ScriptGCThingType::RegExp: + case ScriptGCThingType::Object: + case ScriptGCThingType::Function: + // No-op because GCCellPtr tag bits are zero for objects. + static_assert(uintptr_t(TraceKind::Object) == 0, + "Unexpected tag bits for object GCCellPtr"); + break; + case ScriptGCThingType::BigInt: + // Use xorPtr with a 32-bit immediate because it's more efficient than + // andPtr on 64-bit. + static_assert(uintptr_t(TraceKind::BigInt) == 1, + "Unexpected tag bits for BigInt GCCellPtr"); + masm.xorPtr(Imm32(1), dest); + break; + case ScriptGCThingType::Scope: + // Use xorPtr with a 32-bit immediate because it's more efficient than + // andPtr on 64-bit. + static_assert(uintptr_t(TraceKind::Scope) >= JS::OutOfLineTraceKindMask, + "Expected Scopes to have OutOfLineTraceKindMask tag"); + masm.xorPtr(Imm32(JS::OutOfLineTraceKindMask), dest); + break; + } + +#ifdef DEBUG + // Assert low bits are not set. + Label ok; + masm.branchTestPtr(Assembler::Zero, dest, Imm32(0b111), &ok); + masm.assumeUnreachable("GC pointer with tag bits set"); + masm.bind(&ok); +#endif +} + +template <> +void BaselineCompilerCodeGen::pushScriptGCThingArg(ScriptGCThingType type, + Register scratch1, + Register scratch2) { + gc::Cell* thing = GetScriptGCThing(handler.script(), handler.pc(), type); + pushArg(ImmGCPtr(thing)); +} + +template <> +void BaselineInterpreterCodeGen::pushScriptGCThingArg(ScriptGCThingType type, + Register scratch1, + Register scratch2) { + loadScriptGCThing(type, scratch1, scratch2); + pushArg(scratch1); +} + +template +void BaselineCodeGen::pushScriptNameArg(Register scratch1, + Register scratch2) { + pushScriptGCThingArg(ScriptGCThingType::Atom, scratch1, scratch2); +} + +template <> +void BaselineCompilerCodeGen::pushUint8BytecodeOperandArg(Register) { + MOZ_ASSERT(JOF_OPTYPE(JSOp(*handler.pc())) == JOF_UINT8); + pushArg(Imm32(GET_UINT8(handler.pc()))); +} + +template <> +void BaselineInterpreterCodeGen::pushUint8BytecodeOperandArg(Register scratch) { + LoadUint8Operand(masm, scratch); + pushArg(scratch); +} + +template <> +void BaselineCompilerCodeGen::pushUint16BytecodeOperandArg(Register) { + MOZ_ASSERT(JOF_OPTYPE(JSOp(*handler.pc())) == JOF_UINT16); + pushArg(Imm32(GET_UINT16(handler.pc()))); +} + +template <> +void BaselineInterpreterCodeGen::pushUint16BytecodeOperandArg( + Register scratch) { + LoadUint16Operand(masm, scratch); + pushArg(scratch); +} + +template <> +void BaselineCompilerCodeGen::loadInt32LengthBytecodeOperand(Register dest) { + uint32_t length = GET_UINT32(handler.pc()); + MOZ_ASSERT(length <= INT32_MAX, + "the bytecode emitter must fail to compile code that would " + "produce a length exceeding int32_t range"); + masm.move32(Imm32(AssertedCast(length)), dest); +} + +template <> +void BaselineInterpreterCodeGen::loadInt32LengthBytecodeOperand(Register dest) { + LoadInt32Operand(masm, dest); +} + +template +bool BaselineCodeGen::emitDebugPrologue() { + auto ifDebuggee = [this]() { + // Load pointer to BaselineFrame in R0. + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + + prepareVMCall(); + pushArg(R0.scratchReg()); + + const RetAddrEntry::Kind kind = RetAddrEntry::Kind::DebugPrologue; + + using Fn = bool (*)(JSContext*, BaselineFrame*); + if (!callVM(kind)) { + return false; + } + + return true; + }; + return emitDebugInstrumentation(ifDebuggee); +} + +template <> +void BaselineCompilerCodeGen::emitInitFrameFields(Register nonFunctionEnv) { + Register scratch = R0.scratchReg(); + Register scratch2 = R2.scratchReg(); + MOZ_ASSERT(nonFunctionEnv != scratch && nonFunctionEnv != scratch2); + + masm.store32(Imm32(0), frame.addressOfFlags()); + if (handler.function()) { + masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(), scratch); + masm.unboxObject(Address(scratch, JSFunction::offsetOfEnvironment()), + scratch); + masm.storePtr(scratch, frame.addressOfEnvironmentChain()); + } else { + masm.storePtr(nonFunctionEnv, frame.addressOfEnvironmentChain()); + } + + // If cx->inlinedICScript contains an inlined ICScript (passed from + // the caller), take that ICScript and store it in the frame, then + // overwrite cx->inlinedICScript with nullptr. + Label notInlined, done; + masm.movePtr(ImmPtr(cx->addressOfInlinedICScript()), scratch); + Address inlinedAddr(scratch, 0); + masm.branchPtr(Assembler::Equal, inlinedAddr, ImmWord(0), ¬Inlined); + masm.loadPtr(inlinedAddr, scratch2); + masm.storePtr(scratch2, frame.addressOfICScript()); + masm.storePtr(ImmPtr(nullptr), inlinedAddr); + masm.jump(&done); + + // Otherwise, store this script's default ICSCript in the frame. + masm.bind(¬Inlined); + masm.storePtr(ImmPtr(handler.script()->jitScript()->icScript()), + frame.addressOfICScript()); + masm.bind(&done); +} + +template <> +void BaselineInterpreterCodeGen::emitInitFrameFields(Register nonFunctionEnv) { + MOZ_ASSERT(nonFunctionEnv == R1.scratchReg(), + "Don't clobber nonFunctionEnv below"); + + // If we have a dedicated PC register we use it as scratch1 to avoid a + // register move below. + Register scratch1 = + HasInterpreterPCReg() ? InterpreterPCReg : R0.scratchReg(); + Register scratch2 = R2.scratchReg(); + + masm.store32(Imm32(BaselineFrame::RUNNING_IN_INTERPRETER), + frame.addressOfFlags()); + + // Initialize interpreterScript. + Label notFunction, done; + masm.loadPtr(frame.addressOfCalleeToken(), scratch1); + masm.branchTestPtr(Assembler::NonZero, scratch1, Imm32(CalleeTokenScriptBit), + ¬Function); + { + // CalleeToken_Function or CalleeToken_FunctionConstructing. + masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), scratch1); + masm.unboxObject(Address(scratch1, JSFunction::offsetOfEnvironment()), + scratch2); + masm.storePtr(scratch2, frame.addressOfEnvironmentChain()); + masm.loadPrivate(Address(scratch1, JSFunction::offsetOfJitInfoOrScript()), + scratch1); + masm.jump(&done); + } + masm.bind(¬Function); + { + // CalleeToken_Script. + masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), scratch1); + masm.storePtr(nonFunctionEnv, frame.addressOfEnvironmentChain()); + } + masm.bind(&done); + masm.storePtr(scratch1, frame.addressOfInterpreterScript()); + + // Initialize icScript and interpreterICEntry + masm.loadJitScript(scratch1, scratch2); + masm.computeEffectiveAddress(Address(scratch2, JitScript::offsetOfICScript()), + scratch2); + masm.storePtr(scratch2, frame.addressOfICScript()); + masm.computeEffectiveAddress(Address(scratch2, ICScript::offsetOfICEntries()), + scratch2); + masm.storePtr(scratch2, frame.addressOfInterpreterICEntry()); + + // Initialize interpreter pc. + masm.loadPtr(Address(scratch1, JSScript::offsetOfSharedData()), scratch1); + masm.loadPtr(Address(scratch1, SharedImmutableScriptData::offsetOfISD()), + scratch1); + masm.addPtr(Imm32(ImmutableScriptData::offsetOfCode()), scratch1); + + if (HasInterpreterPCReg()) { + MOZ_ASSERT(scratch1 == InterpreterPCReg, + "pc must be stored in the pc register"); + } else { + masm.storePtr(scratch1, frame.addressOfInterpreterPC()); + } +} + +// Assert we don't need a post write barrier to write sourceObj to a slot of +// destObj. See comments in WarpBuilder::buildNamedLambdaEnv. +static void AssertCanElidePostWriteBarrier(MacroAssembler& masm, + Register destObj, Register sourceObj, + Register temp) { +#ifdef DEBUG + Label ok; + masm.branchPtrInNurseryChunk(Assembler::Equal, destObj, temp, &ok); + masm.branchPtrInNurseryChunk(Assembler::NotEqual, sourceObj, temp, &ok); + masm.assumeUnreachable("Unexpected missing post write barrier in Baseline"); + masm.bind(&ok); +#endif +} + +template <> +bool BaselineCompilerCodeGen::initEnvironmentChain() { + if (!handler.function()) { + return true; + } + if (!handler.script()->needsFunctionEnvironmentObjects()) { + return true; + } + + // Allocate a NamedLambdaObject and/or a CallObject. If the function needs + // both, the NamedLambdaObject must enclose the CallObject. If one of the + // allocations fails, we perform the whole operation in C++. + + JSObject* templateEnv = handler.script()->jitScript()->templateEnvironment(); + MOZ_ASSERT(templateEnv); + + CallObject* callObjectTemplate = nullptr; + if (handler.function()->needsCallObject()) { + callObjectTemplate = &templateEnv->as(); + } + + NamedLambdaObject* namedLambdaTemplate = nullptr; + if (handler.function()->needsNamedLambdaEnvironment()) { + if (callObjectTemplate) { + templateEnv = templateEnv->enclosingEnvironment(); + } + namedLambdaTemplate = &templateEnv->as(); + } + + MOZ_ASSERT(namedLambdaTemplate || callObjectTemplate); + + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + Register newEnv = regs.takeAny(); + Register enclosingEnv = regs.takeAny(); + Register callee = regs.takeAny(); + Register temp = regs.takeAny(); + + Label fail; + masm.loadPtr(frame.addressOfEnvironmentChain(), enclosingEnv); + masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(), callee); + + // Allocate a NamedLambdaObject if needed. + if (namedLambdaTemplate) { + TemplateObject templateObject(namedLambdaTemplate); + masm.createGCObject(newEnv, temp, templateObject, gc::Heap::Default, &fail); + + // Store enclosing environment. + Address enclosingSlot(newEnv, + NamedLambdaObject::offsetOfEnclosingEnvironment()); + masm.storeValue(JSVAL_TYPE_OBJECT, enclosingEnv, enclosingSlot); + AssertCanElidePostWriteBarrier(masm, newEnv, enclosingEnv, temp); + + // Store callee. + Address lambdaSlot(newEnv, NamedLambdaObject::offsetOfLambdaSlot()); + masm.storeValue(JSVAL_TYPE_OBJECT, callee, lambdaSlot); + AssertCanElidePostWriteBarrier(masm, newEnv, callee, temp); + + if (callObjectTemplate) { + masm.movePtr(newEnv, enclosingEnv); + } + } + + // Allocate a CallObject if needed. + if (callObjectTemplate) { + TemplateObject templateObject(callObjectTemplate); + masm.createGCObject(newEnv, temp, templateObject, gc::Heap::Default, &fail); + + // Store enclosing environment. + Address enclosingSlot(newEnv, CallObject::offsetOfEnclosingEnvironment()); + masm.storeValue(JSVAL_TYPE_OBJECT, enclosingEnv, enclosingSlot); + AssertCanElidePostWriteBarrier(masm, newEnv, enclosingEnv, temp); + + // Store callee. + Address calleeSlot(newEnv, CallObject::offsetOfCallee()); + masm.storeValue(JSVAL_TYPE_OBJECT, callee, calleeSlot); + AssertCanElidePostWriteBarrier(masm, newEnv, callee, temp); + } + + // Update the frame's environment chain and mark it initialized. + Label done; + masm.storePtr(newEnv, frame.addressOfEnvironmentChain()); + masm.or32(Imm32(BaselineFrame::HAS_INITIAL_ENV), frame.addressOfFlags()); + masm.jump(&done); + + masm.bind(&fail); + + prepareVMCall(); + + masm.loadBaselineFramePtr(FramePointer, temp); + pushArg(temp); + + const CallVMPhase phase = CallVMPhase::BeforePushingLocals; + + using Fn = bool (*)(JSContext*, BaselineFrame*); + if (!callVMNonOp(phase)) { + return false; + } + + masm.bind(&done); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::initEnvironmentChain() { + // For function scripts, call InitFunctionEnvironmentObjects if needed. For + // non-function scripts this is a no-op. + + Label done; + masm.branchTestPtr(Assembler::NonZero, frame.addressOfCalleeToken(), + Imm32(CalleeTokenScriptBit), &done); + { + auto initEnv = [this]() { + // Call into the VM to create the proper environment objects. + prepareVMCall(); + + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + pushArg(R0.scratchReg()); + + const CallVMPhase phase = CallVMPhase::BeforePushingLocals; + + using Fn = bool (*)(JSContext*, BaselineFrame*); + return callVMNonOp(phase); + }; + if (!emitTestScriptFlag( + JSScript::ImmutableFlags::NeedsFunctionEnvironmentObjects, true, + initEnv, R2.scratchReg())) { + return false; + } + } + + masm.bind(&done); + return true; +} + +template +bool BaselineCodeGen::emitInterruptCheck() { + frame.syncStack(0); + + Label done; + masm.branch32(Assembler::Equal, AbsoluteAddress(cx->addressOfInterruptBits()), + Imm32(0), &done); + + prepareVMCall(); + + // Use a custom RetAddrEntry::Kind so DebugModeOSR can distinguish this call + // from other callVMs that might happen at this pc. + const RetAddrEntry::Kind kind = RetAddrEntry::Kind::InterruptCheck; + + using Fn = bool (*)(JSContext*); + if (!callVM(kind)) { + return false; + } + + masm.bind(&done); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emitWarmUpCounterIncrement() { + frame.assertSyncedStack(); + + // Record native code offset for OSR from Baseline Interpreter into Baseline + // JIT code. This is right before the warm-up check in the Baseline JIT code, + // to make sure we can immediately enter Ion if the script is warm enough or + // if --ion-eager is used. + JSScript* script = handler.script(); + jsbytecode* pc = handler.pc(); + if (JSOp(*pc) == JSOp::LoopHead) { + uint32_t pcOffset = script->pcToOffset(pc); + uint32_t nativeOffset = masm.currentOffset(); + if (!handler.osrEntries().emplaceBack(pcOffset, nativeOffset)) { + ReportOutOfMemory(cx); + return false; + } + } + + // Emit no warm-up counter increments if Ion is not enabled or if the script + // will never be Ion-compileable. + if (!handler.maybeIonCompileable()) { + return true; + } + + Register scriptReg = R2.scratchReg(); + Register countReg = R0.scratchReg(); + + // Load the ICScript* in scriptReg. + masm.loadPtr(frame.addressOfICScript(), scriptReg); + + // Bump warm-up counter. + Address warmUpCounterAddr(scriptReg, ICScript::offsetOfWarmUpCount()); + masm.load32(warmUpCounterAddr, countReg); + masm.add32(Imm32(1), countReg); + masm.store32(countReg, warmUpCounterAddr); + + if (!JitOptions.disableInlining) { + // Consider trial inlining. + // Note: unlike other warmup thresholds, where we try to enter a + // higher tier whenever we are higher than a given warmup count, + // trial inlining triggers once when reaching the threshold. + Label noTrialInlining; + masm.branch32(Assembler::NotEqual, countReg, + Imm32(JitOptions.trialInliningWarmUpThreshold), + &noTrialInlining); + prepareVMCall(); + + masm.PushBaselineFramePtr(FramePointer, R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*); + if (!callVMNonOp()) { + return false; + } + // Reload registers potentially clobbered by the call. + masm.loadPtr(frame.addressOfICScript(), scriptReg); + masm.load32(warmUpCounterAddr, countReg); + masm.bind(&noTrialInlining); + } + + if (JSOp(*pc) == JSOp::LoopHead) { + // If this is a loop where we can't OSR (for example because it's inside a + // catch or finally block), increment the warmup counter but don't attempt + // OSR (Ion/Warp only compiles the try block). + if (!handler.analysis().info(pc).loopHeadCanOsr) { + return true; + } + } + + Label done; + + const OptimizationInfo* info = + IonOptimizations.get(OptimizationLevel::Normal); + uint32_t warmUpThreshold = info->compilerWarmUpThreshold(script, pc); + masm.branch32(Assembler::LessThan, countReg, Imm32(warmUpThreshold), &done); + + // Don't trigger Warp compilations from trial-inlined scripts. + Address depthAddr(scriptReg, ICScript::offsetOfDepth()); + masm.branch32(Assembler::NotEqual, depthAddr, Imm32(0), &done); + + // Load the IonScript* in scriptReg. We can load this from the ICScript* + // because it must be an outer ICScript embedded in the JitScript. + constexpr int32_t offset = -int32_t(JitScript::offsetOfICScript()) + + int32_t(JitScript::offsetOfIonScript()); + masm.loadPtr(Address(scriptReg, offset), scriptReg); + + // Do nothing if Ion is already compiling this script off-thread or if Ion has + // been disabled for this script. + masm.branchPtr(Assembler::Equal, scriptReg, ImmPtr(IonCompilingScriptPtr), + &done); + masm.branchPtr(Assembler::Equal, scriptReg, ImmPtr(IonDisabledScriptPtr), + &done); + + // Try to compile and/or finish a compilation. + if (JSOp(*pc) == JSOp::LoopHead) { + // Try to OSR into Ion. + computeFrameSize(R0.scratchReg()); + + prepareVMCall(); + + pushBytecodePCArg(); + pushArg(R0.scratchReg()); + masm.PushBaselineFramePtr(FramePointer, R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, uint32_t, jsbytecode*, + IonOsrTempData**); + if (!callVM()) { + return false; + } + + // The return register holds the IonOsrTempData*. Perform OSR if it's not + // nullptr. + static_assert(ReturnReg != OsrFrameReg, + "Code below depends on osrDataReg != OsrFrameReg"); + Register osrDataReg = ReturnReg; + masm.branchTestPtr(Assembler::Zero, osrDataReg, osrDataReg, &done); + + // Success! Switch from Baseline JIT code to Ion JIT code. + + // At this point, stack looks like: + // + // +-> [...Calling-Frame...] + // | [...Actual-Args/ThisV/ArgCount/Callee...] + // | [Descriptor] + // | [Return-Addr] + // +---[Saved-FramePtr] + // [...Baseline-Frame...] + +#ifdef DEBUG + // Get a scratch register that's not osrDataReg or OsrFrameReg. + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + MOZ_ASSERT(!regs.has(FramePointer)); + regs.take(osrDataReg); + regs.take(OsrFrameReg); + + Register scratchReg = regs.takeAny(); + + // If profiler instrumentation is on, ensure that lastProfilingFrame is + // the frame currently being OSR-ed + { + Label checkOk; + AbsoluteAddress addressOfEnabled( + cx->runtime()->geckoProfiler().addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &checkOk); + masm.loadPtr(AbsoluteAddress((void*)&cx->jitActivation), scratchReg); + masm.loadPtr( + Address(scratchReg, JitActivation::offsetOfLastProfilingFrame()), + scratchReg); + + // It may be the case that we entered the baseline frame with + // profiling turned off on, then in a call within a loop (i.e. a + // callee frame), turn on profiling, then return to this frame, + // and then OSR with profiling turned on. In this case, allow for + // lastProfilingFrame to be null. + masm.branchPtr(Assembler::Equal, scratchReg, ImmWord(0), &checkOk); + + masm.branchPtr(Assembler::Equal, FramePointer, scratchReg, &checkOk); + masm.assumeUnreachable("Baseline OSR lastProfilingFrame mismatch."); + masm.bind(&checkOk); + } +#endif + + // Restore the stack pointer so that the saved frame pointer is on top of + // the stack. + masm.moveToStackPtr(FramePointer); + + // Jump into Ion. + masm.loadPtr(Address(osrDataReg, IonOsrTempData::offsetOfBaselineFrame()), + OsrFrameReg); + masm.jump(Address(osrDataReg, IonOsrTempData::offsetOfJitCode())); + } else { + prepareVMCall(); + + masm.PushBaselineFramePtr(FramePointer, R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*); + if (!callVMNonOp()) { + return false; + } + } + + masm.bind(&done); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emitWarmUpCounterIncrement() { + Register scriptReg = R2.scratchReg(); + Register countReg = R0.scratchReg(); + + // Load the JitScript* in scriptReg. + loadScript(scriptReg); + masm.loadJitScript(scriptReg, scriptReg); + + // Bump warm-up counter. + Address warmUpCounterAddr(scriptReg, JitScript::offsetOfWarmUpCount()); + masm.load32(warmUpCounterAddr, countReg); + masm.add32(Imm32(1), countReg); + masm.store32(countReg, warmUpCounterAddr); + + // If the script is warm enough for Baseline compilation, call into the VM to + // compile it. + Label done; + masm.branch32(Assembler::BelowOrEqual, countReg, + Imm32(JitOptions.baselineJitWarmUpThreshold), &done); + masm.branchPtr(Assembler::Equal, + Address(scriptReg, JitScript::offsetOfBaselineScript()), + ImmPtr(BaselineDisabledScriptPtr), &done); + { + prepareVMCall(); + + masm.PushBaselineFramePtr(FramePointer, R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, uint8_t**); + if (!callVM()) { + return false; + } + + // If the function returned nullptr we either skipped compilation or were + // unable to compile the script. Continue running in the interpreter. + masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &done); + + // Success! Switch from interpreter to JIT code by jumping to the + // corresponding code in the BaselineScript. + // + // This works because BaselineCompiler uses the same frame layout (stack is + // synced at OSR points) and BaselineCompileFromBaselineInterpreter has + // already cleared the RUNNING_IN_INTERPRETER flag for us. + // See BaselineFrame::prepareForBaselineInterpreterToJitOSR. + masm.jump(ReturnReg); + } + + masm.bind(&done); + return true; +} + +bool BaselineCompiler::emitDebugTrap() { + MOZ_ASSERT(compileDebugInstrumentation()); + MOZ_ASSERT(frame.numUnsyncedSlots() == 0); + + JSScript* script = handler.script(); + bool enabled = DebugAPI::stepModeEnabled(script) || + DebugAPI::hasBreakpointsAt(script, handler.pc()); + + // Emit patchable call to debug trap handler. + JitCode* handlerCode = cx->runtime()->jitRuntime()->debugTrapHandler( + cx, DebugTrapHandlerKind::Compiler); + if (!handlerCode) { + return false; + } + + CodeOffset nativeOffset = masm.toggledCall(handlerCode, enabled); + + uint32_t pcOffset = script->pcToOffset(handler.pc()); + if (!debugTrapEntries_.emplaceBack(pcOffset, nativeOffset.offset())) { + ReportOutOfMemory(cx); + return false; + } + + // Add a RetAddrEntry for the return offset -> pc mapping. + return handler.recordCallRetAddr(cx, RetAddrEntry::Kind::DebugTrap, + masm.currentOffset()); +} + +template +void BaselineCodeGen::emitProfilerEnterFrame() { + // Store stack position to lastProfilingFrame variable, guarded by a toggled + // jump. Starts off initially disabled. + Label noInstrument; + CodeOffset toggleOffset = masm.toggledJump(&noInstrument); + masm.profilerEnterFrame(FramePointer, R0.scratchReg()); + masm.bind(&noInstrument); + + // Store the start offset in the appropriate location. + MOZ_ASSERT(!profilerEnterFrameToggleOffset_.bound()); + profilerEnterFrameToggleOffset_ = toggleOffset; +} + +template +void BaselineCodeGen::emitProfilerExitFrame() { + // Store previous frame to lastProfilingFrame variable, guarded by a toggled + // jump. Starts off initially disabled. + Label noInstrument; + CodeOffset toggleOffset = masm.toggledJump(&noInstrument); + masm.profilerExitFrame(); + masm.bind(&noInstrument); + + // Store the start offset in the appropriate location. + MOZ_ASSERT(!profilerExitFrameToggleOffset_.bound()); + profilerExitFrameToggleOffset_ = toggleOffset; +} + +template +bool BaselineCodeGen::emit_Nop() { + return true; +} + +template +bool BaselineCodeGen::emit_NopDestructuring() { + return true; +} + +template +bool BaselineCodeGen::emit_TryDestructuring() { + return true; +} + +template +bool BaselineCodeGen::emit_Pop() { + frame.pop(); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_PopN() { + frame.popn(GET_UINT16(handler.pc())); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_PopN() { + LoadUint16Operand(masm, R0.scratchReg()); + frame.popn(R0.scratchReg()); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_DupAt() { + frame.syncStack(0); + + // DupAt takes a value on the stack and re-pushes it on top. It's like + // GetLocal but it addresses from the top of the stack instead of from the + // stack frame. + + int depth = -(GET_UINT24(handler.pc()) + 1); + masm.loadValue(frame.addressOfStackValue(depth), R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_DupAt() { + LoadUint24Operand(masm, 0, R0.scratchReg()); + masm.loadValue(frame.addressOfStackValue(R0.scratchReg()), R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_Dup() { + // Keep top stack value in R0, sync the rest so that we can use R1. We use + // separate registers because every register can be used by at most one + // StackValue. + frame.popRegsAndSync(1); + masm.moveValue(R0, R1); + + // inc/dec ops use Dup followed by Inc/Dec. Push R0 last to avoid a move. + frame.push(R1); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_Dup2() { + frame.syncStack(0); + + masm.loadValue(frame.addressOfStackValue(-2), R0); + masm.loadValue(frame.addressOfStackValue(-1), R1); + + frame.push(R0); + frame.push(R1); + return true; +} + +template +bool BaselineCodeGen::emit_Swap() { + // Keep top stack values in R0 and R1. + frame.popRegsAndSync(2); + + frame.push(R1); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_Pick() { + frame.syncStack(0); + + // Pick takes a value on the stack and moves it to the top. + // For instance, pick 2: + // before: A B C D E + // after : A B D E C + + // First, move value at -(amount + 1) into R0. + int32_t depth = -(GET_INT8(handler.pc()) + 1); + masm.loadValue(frame.addressOfStackValue(depth), R0); + + // Move the other values down. + depth++; + for (; depth < 0; depth++) { + Address source = frame.addressOfStackValue(depth); + Address dest = frame.addressOfStackValue(depth - 1); + masm.loadValue(source, R1); + masm.storeValue(R1, dest); + } + + // Push R0. + frame.pop(); + frame.push(R0); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_Pick() { + // First, move the value to move up into R0. + Register scratch = R2.scratchReg(); + LoadUint8Operand(masm, scratch); + masm.loadValue(frame.addressOfStackValue(scratch), R0); + + // Move the other values down. + Label top, done; + masm.bind(&top); + masm.branchSub32(Assembler::Signed, Imm32(1), scratch, &done); + { + masm.loadValue(frame.addressOfStackValue(scratch), R1); + masm.storeValue(R1, frame.addressOfStackValue(scratch, sizeof(Value))); + masm.jump(&top); + } + + masm.bind(&done); + + // Replace value on top of the stack with R0. + masm.storeValue(R0, frame.addressOfStackValue(-1)); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_Unpick() { + frame.syncStack(0); + + // Pick takes the top of the stack value and moves it under the nth value. + // For instance, unpick 2: + // before: A B C D E + // after : A B E C D + + // First, move value at -1 into R0. + masm.loadValue(frame.addressOfStackValue(-1), R0); + + MOZ_ASSERT(GET_INT8(handler.pc()) > 0, + "Interpreter code assumes JSOp::Unpick operand > 0"); + + // Move the other values up. + int32_t depth = -(GET_INT8(handler.pc()) + 1); + for (int32_t i = -1; i > depth; i--) { + Address source = frame.addressOfStackValue(i - 1); + Address dest = frame.addressOfStackValue(i); + masm.loadValue(source, R1); + masm.storeValue(R1, dest); + } + + // Store R0 under the nth value. + Address dest = frame.addressOfStackValue(depth); + masm.storeValue(R0, dest); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_Unpick() { + Register scratch = R2.scratchReg(); + LoadUint8Operand(masm, scratch); + + // Move the top value into R0. + masm.loadValue(frame.addressOfStackValue(-1), R0); + + // Overwrite the nth stack value with R0 but first save the old value in R1. + masm.loadValue(frame.addressOfStackValue(scratch), R1); + masm.storeValue(R0, frame.addressOfStackValue(scratch)); + + // Now for each slot x in [n-1, 1] do the following: + // + // * Store the value in slot x in R0. + // * Store the value in the previous slot (now in R1) in slot x. + // * Move R0 to R1. + +#ifdef DEBUG + // Assert the operand > 0 so the branchSub32 below doesn't "underflow" to + // negative values. + { + Label ok; + masm.branch32(Assembler::GreaterThan, scratch, Imm32(0), &ok); + masm.assumeUnreachable("JSOp::Unpick with operand <= 0?"); + masm.bind(&ok); + } +#endif + + Label top, done; + masm.bind(&top); + masm.branchSub32(Assembler::Zero, Imm32(1), scratch, &done); + { + // Overwrite stack slot x with slot x + 1, saving the old value in R1. + masm.loadValue(frame.addressOfStackValue(scratch), R0); + masm.storeValue(R1, frame.addressOfStackValue(scratch)); + masm.moveValue(R0, R1); + masm.jump(&top); + } + + // Finally, replace the value on top of the stack (slot 0) with R1. This is + // the value that used to be in slot 1. + masm.bind(&done); + masm.storeValue(R1, frame.addressOfStackValue(-1)); + return true; +} + +template <> +void BaselineCompilerCodeGen::emitJump() { + jsbytecode* pc = handler.pc(); + MOZ_ASSERT(IsJumpOpcode(JSOp(*pc))); + frame.assertSyncedStack(); + + jsbytecode* target = pc + GET_JUMP_OFFSET(pc); + masm.jump(handler.labelOf(target)); +} + +template <> +void BaselineInterpreterCodeGen::emitJump() { + // We have to add the current pc's jump offset to the current pc. We can use + // R0 and R1 as scratch because we jump to the "next op" label so these + // registers aren't in use at this point. + Register scratch1 = R0.scratchReg(); + Register scratch2 = R1.scratchReg(); + Register pc = LoadBytecodePC(masm, scratch1); + LoadInt32OperandSignExtendToPtr(masm, pc, scratch2); + if (HasInterpreterPCReg()) { + masm.addPtr(scratch2, InterpreterPCReg); + } else { + masm.addPtr(pc, scratch2); + masm.storePtr(scratch2, frame.addressOfInterpreterPC()); + } + masm.jump(handler.interpretOpWithPCRegLabel()); +} + +template <> +void BaselineCompilerCodeGen::emitTestBooleanTruthy(bool branchIfTrue, + ValueOperand val) { + jsbytecode* pc = handler.pc(); + MOZ_ASSERT(IsJumpOpcode(JSOp(*pc))); + frame.assertSyncedStack(); + + jsbytecode* target = pc + GET_JUMP_OFFSET(pc); + masm.branchTestBooleanTruthy(branchIfTrue, val, handler.labelOf(target)); +} + +template <> +void BaselineInterpreterCodeGen::emitTestBooleanTruthy(bool branchIfTrue, + ValueOperand val) { + Label done; + masm.branchTestBooleanTruthy(!branchIfTrue, val, &done); + emitJump(); + masm.bind(&done); +} + +template <> +template +[[nodiscard]] bool BaselineCompilerCodeGen::emitTestScriptFlag( + JSScript::ImmutableFlags flag, const F1& ifSet, const F2& ifNotSet, + Register scratch) { + if (handler.script()->hasFlag(flag)) { + return ifSet(); + } + return ifNotSet(); +} + +template <> +template +[[nodiscard]] bool BaselineInterpreterCodeGen::emitTestScriptFlag( + JSScript::ImmutableFlags flag, const F1& ifSet, const F2& ifNotSet, + Register scratch) { + Label flagNotSet, done; + loadScript(scratch); + masm.branchTest32(Assembler::Zero, + Address(scratch, JSScript::offsetOfImmutableFlags()), + Imm32(uint32_t(flag)), &flagNotSet); + { + if (!ifSet()) { + return false; + } + masm.jump(&done); + } + masm.bind(&flagNotSet); + { + if (!ifNotSet()) { + return false; + } + } + + masm.bind(&done); + return true; +} + +template <> +template +[[nodiscard]] bool BaselineCompilerCodeGen::emitTestScriptFlag( + JSScript::ImmutableFlags flag, bool value, const F& emit, + Register scratch) { + if (handler.script()->hasFlag(flag) == value) { + return emit(); + } + return true; +} + +template <> +template +[[nodiscard]] bool BaselineCompilerCodeGen::emitTestScriptFlag( + JSScript::MutableFlags flag, bool value, const F& emit, Register scratch) { + if (handler.script()->hasFlag(flag) == value) { + return emit(); + } + return true; +} + +template <> +template +[[nodiscard]] bool BaselineInterpreterCodeGen::emitTestScriptFlag( + JSScript::ImmutableFlags flag, bool value, const F& emit, + Register scratch) { + Label done; + loadScript(scratch); + masm.branchTest32(value ? Assembler::Zero : Assembler::NonZero, + Address(scratch, JSScript::offsetOfImmutableFlags()), + Imm32(uint32_t(flag)), &done); + { + if (!emit()) { + return false; + } + } + + masm.bind(&done); + return true; +} + +template <> +template +[[nodiscard]] bool BaselineInterpreterCodeGen::emitTestScriptFlag( + JSScript::MutableFlags flag, bool value, const F& emit, Register scratch) { + Label done; + loadScript(scratch); + masm.branchTest32(value ? Assembler::Zero : Assembler::NonZero, + Address(scratch, JSScript::offsetOfMutableFlags()), + Imm32(uint32_t(flag)), &done); + { + if (!emit()) { + return false; + } + } + + masm.bind(&done); + return true; +} + +template +bool BaselineCodeGen::emit_Goto() { + frame.syncStack(0); + emitJump(); + return true; +} + +template +bool BaselineCodeGen::emitTest(bool branchIfTrue) { + bool knownBoolean = frame.stackValueHasKnownType(-1, JSVAL_TYPE_BOOLEAN); + + // Keep top stack value in R0. + frame.popRegsAndSync(1); + + if (!knownBoolean && !emitNextIC()) { + return false; + } + + // IC will leave a BooleanValue in R0, just need to branch on it. + emitTestBooleanTruthy(branchIfTrue, R0); + return true; +} + +template +bool BaselineCodeGen::emit_JumpIfFalse() { + return emitTest(false); +} + +template +bool BaselineCodeGen::emit_JumpIfTrue() { + return emitTest(true); +} + +template +bool BaselineCodeGen::emitAndOr(bool branchIfTrue) { + bool knownBoolean = frame.stackValueHasKnownType(-1, JSVAL_TYPE_BOOLEAN); + + // And and Or leave the original value on the stack. + frame.syncStack(0); + + masm.loadValue(frame.addressOfStackValue(-1), R0); + if (!knownBoolean && !emitNextIC()) { + return false; + } + + emitTestBooleanTruthy(branchIfTrue, R0); + return true; +} + +template +bool BaselineCodeGen::emit_And() { + return emitAndOr(false); +} + +template +bool BaselineCodeGen::emit_Or() { + return emitAndOr(true); +} + +template +bool BaselineCodeGen::emit_Coalesce() { + // Coalesce leaves the original value on the stack. + frame.syncStack(0); + + masm.loadValue(frame.addressOfStackValue(-1), R0); + + Label undefinedOrNull; + + masm.branchTestUndefined(Assembler::Equal, R0, &undefinedOrNull); + masm.branchTestNull(Assembler::Equal, R0, &undefinedOrNull); + emitJump(); + + masm.bind(&undefinedOrNull); + // fall through + return true; +} + +template +bool BaselineCodeGen::emit_Not() { + bool knownBoolean = frame.stackValueHasKnownType(-1, JSVAL_TYPE_BOOLEAN); + + // Keep top stack value in R0. + frame.popRegsAndSync(1); + + if (!knownBoolean && !emitNextIC()) { + return false; + } + + masm.notBoolean(R0); + + frame.push(R0, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_Pos() { + return emitUnaryArith(); +} + +template +bool BaselineCodeGen::emit_ToNumeric() { + return emitUnaryArith(); +} + +template +bool BaselineCodeGen::emit_LoopHead() { + if (!emit_JumpTarget()) { + return false; + } + if (!emitInterruptCheck()) { + return false; + } + if (!emitWarmUpCounterIncrement()) { + return false; + } + return true; +} + +template +bool BaselineCodeGen::emit_Void() { + frame.pop(); + frame.push(UndefinedValue()); + return true; +} + +template +bool BaselineCodeGen::emit_Undefined() { + frame.push(UndefinedValue()); + return true; +} + +template +bool BaselineCodeGen::emit_Hole() { + frame.push(MagicValue(JS_ELEMENTS_HOLE)); + return true; +} + +template +bool BaselineCodeGen::emit_Null() { + frame.push(NullValue()); + return true; +} + +template +bool BaselineCodeGen::emit_CheckIsObj() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-1), R0); + + Label ok; + masm.branchTestObject(Assembler::Equal, R0, &ok); + + prepareVMCall(); + + pushUint8BytecodeOperandArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, CheckIsObjectKind); + if (!callVM()) { + return false; + } + + masm.bind(&ok); + return true; +} + +template +bool BaselineCodeGen::emit_CheckThis() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-1), R0); + + return emitCheckThis(R0); +} + +template +bool BaselineCodeGen::emit_CheckThisReinit() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-1), R0); + + return emitCheckThis(R0, /* reinit = */ true); +} + +template +bool BaselineCodeGen::emitCheckThis(ValueOperand val, bool reinit) { + Label thisOK; + if (reinit) { + masm.branchTestMagic(Assembler::Equal, val, &thisOK); + } else { + masm.branchTestMagic(Assembler::NotEqual, val, &thisOK); + } + + prepareVMCall(); + + if (reinit) { + using Fn = bool (*)(JSContext*); + if (!callVM()) { + return false; + } + } else { + using Fn = bool (*)(JSContext*); + if (!callVM()) { + return false; + } + } + + masm.bind(&thisOK); + return true; +} + +template +bool BaselineCodeGen::emit_CheckReturn() { + MOZ_ASSERT_IF(handler.maybeScript(), + handler.maybeScript()->isDerivedClassConstructor()); + + // Load |this| in R0, return value in R1. + frame.popRegsAndSync(1); + emitLoadReturnValue(R1); + + Label done, returnBad, checkThis; + masm.branchTestObject(Assembler::NotEqual, R1, &checkThis); + { + masm.moveValue(R1, R0); + masm.jump(&done); + } + masm.bind(&checkThis); + masm.branchTestUndefined(Assembler::NotEqual, R1, &returnBad); + masm.branchTestMagic(Assembler::NotEqual, R0, &done); + masm.bind(&returnBad); + + prepareVMCall(); + pushArg(R1); + + using Fn = bool (*)(JSContext*, HandleValue); + if (!callVM()) { + return false; + } + masm.assumeUnreachable("Should throw on bad derived constructor return"); + + masm.bind(&done); + + // Push |rval| or |this| onto the stack. + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_FunctionThis() { + MOZ_ASSERT_IF(handler.maybeFunction(), !handler.maybeFunction()->isArrow()); + + frame.pushThis(); + + auto boxThis = [this]() { + // Load |thisv| in R0. Skip the call if it's already an object. + Label skipCall; + frame.popRegsAndSync(1); + masm.branchTestObject(Assembler::Equal, R0, &skipCall); + + prepareVMCall(); + masm.loadBaselineFramePtr(FramePointer, R1.scratchReg()); + + pushArg(R1.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, MutableHandleValue); + if (!callVM()) { + return false; + } + + masm.bind(&skipCall); + frame.push(R0); + return true; + }; + + // In strict mode code, |this| is left alone. + return emitTestScriptFlag(JSScript::ImmutableFlags::Strict, false, boxThis, + R2.scratchReg()); +} + +template +bool BaselineCodeGen::emit_GlobalThis() { + frame.syncStack(0); + + loadGlobalThisValue(R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_NonSyntacticGlobalThis() { + frame.syncStack(0); + + prepareVMCall(); + + masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg()); + pushArg(R0.scratchReg()); + + using Fn = void (*)(JSContext*, HandleObject, MutableHandleValue); + if (!callVM()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_True() { + frame.push(BooleanValue(true)); + return true; +} + +template +bool BaselineCodeGen::emit_False() { + frame.push(BooleanValue(false)); + return true; +} + +template +bool BaselineCodeGen::emit_Zero() { + frame.push(Int32Value(0)); + return true; +} + +template +bool BaselineCodeGen::emit_One() { + frame.push(Int32Value(1)); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_Int8() { + frame.push(Int32Value(GET_INT8(handler.pc()))); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_Int8() { + LoadInt8Operand(masm, R0.scratchReg()); + masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_Int32() { + frame.push(Int32Value(GET_INT32(handler.pc()))); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_Int32() { + LoadInt32Operand(masm, R0.scratchReg()); + masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_Uint16() { + frame.push(Int32Value(GET_UINT16(handler.pc()))); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_Uint16() { + LoadUint16Operand(masm, R0.scratchReg()); + masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_Uint24() { + frame.push(Int32Value(GET_UINT24(handler.pc()))); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_Uint24() { + LoadUint24Operand(masm, 0, R0.scratchReg()); + masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_Double() { + frame.push(GET_INLINE_VALUE(handler.pc())); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_Double() { + LoadInlineValueOperand(masm, R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_BigInt() { + BigInt* bi = handler.script()->getBigInt(handler.pc()); + frame.push(BigIntValue(bi)); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_BigInt() { + Register scratch1 = R0.scratchReg(); + Register scratch2 = R1.scratchReg(); + loadScriptGCThing(ScriptGCThingType::BigInt, scratch1, scratch2); + masm.tagValue(JSVAL_TYPE_BIGINT, scratch1, R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_String() { + frame.push(StringValue(handler.script()->getString(handler.pc()))); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_String() { + Register scratch1 = R0.scratchReg(); + Register scratch2 = R1.scratchReg(); + loadScriptGCThing(ScriptGCThingType::String, scratch1, scratch2); + masm.tagValue(JSVAL_TYPE_STRING, scratch1, R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_Symbol() { + unsigned which = GET_UINT8(handler.pc()); + JS::Symbol* sym = cx->runtime()->wellKnownSymbols->get(which); + frame.push(SymbolValue(sym)); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_Symbol() { + Register scratch1 = R0.scratchReg(); + Register scratch2 = R1.scratchReg(); + LoadUint8Operand(masm, scratch1); + + masm.movePtr(ImmPtr(cx->runtime()->wellKnownSymbols), scratch2); + masm.loadPtr(BaseIndex(scratch2, scratch1, ScalePointer), scratch1); + + masm.tagValue(JSVAL_TYPE_SYMBOL, scratch1, R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_Object() { + frame.push(ObjectValue(*handler.script()->getObject(handler.pc()))); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_Object() { + Register scratch1 = R0.scratchReg(); + Register scratch2 = R1.scratchReg(); + loadScriptGCThing(ScriptGCThingType::Object, scratch1, scratch2); + masm.tagValue(JSVAL_TYPE_OBJECT, scratch1, R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_CallSiteObj() { + return emit_Object(); +} + +template +bool BaselineCodeGen::emit_RegExp() { + prepareVMCall(); + pushScriptGCThingArg(ScriptGCThingType::RegExp, R0.scratchReg(), + R1.scratchReg()); + + using Fn = JSObject* (*)(JSContext*, Handle); + if (!callVM()) { + return false; + } + + // Box and push return value. + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.push(R0); + return true; +} + +#ifdef ENABLE_RECORD_TUPLE +# define UNSUPPORTED_OPCODE(OP) \ + template \ + bool BaselineCodeGen::emit_##OP() { \ + MOZ_CRASH("Record and Tuple are not supported by jit"); \ + return false; \ + } + +UNSUPPORTED_OPCODE(InitRecord) +UNSUPPORTED_OPCODE(AddRecordProperty) +UNSUPPORTED_OPCODE(AddRecordSpread) +UNSUPPORTED_OPCODE(FinishRecord) +UNSUPPORTED_OPCODE(InitTuple) +UNSUPPORTED_OPCODE(AddTupleElement) +UNSUPPORTED_OPCODE(FinishTuple) + +# undef UNSUPPORTED_OPCODE +#endif + +template +bool BaselineCodeGen::emit_Lambda() { + prepareVMCall(); + masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg()); + + pushArg(R0.scratchReg()); + pushScriptGCThingArg(ScriptGCThingType::Function, R0.scratchReg(), + R1.scratchReg()); + + using Fn = JSObject* (*)(JSContext*, HandleFunction, HandleObject); + if (!callVM()) { + return false; + } + + // Box and push return value. + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_SetFunName() { + frame.popRegsAndSync(2); + + frame.push(R0); + frame.syncStack(0); + + masm.unboxObject(R0, R0.scratchReg()); + + prepareVMCall(); + + pushUint8BytecodeOperandArg(R2.scratchReg()); + pushArg(R1); + pushArg(R0.scratchReg()); + + using Fn = + bool (*)(JSContext*, HandleFunction, HandleValue, FunctionPrefixKind); + return callVM(); +} + +template +bool BaselineCodeGen::emit_BitOr() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_BitXor() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_BitAnd() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_Lsh() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_Rsh() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_Ursh() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_Add() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_Sub() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_Mul() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_Div() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_Mod() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emit_Pow() { + return emitBinaryArith(); +} + +template +bool BaselineCodeGen::emitBinaryArith() { + // Keep top JSStack value in R0 and R2 + frame.popRegsAndSync(2); + + // Call IC + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emitUnaryArith() { + // Keep top stack value in R0. + frame.popRegsAndSync(1); + + // Call IC + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_BitNot() { + return emitUnaryArith(); +} + +template +bool BaselineCodeGen::emit_Neg() { + return emitUnaryArith(); +} + +template +bool BaselineCodeGen::emit_Inc() { + return emitUnaryArith(); +} + +template +bool BaselineCodeGen::emit_Dec() { + return emitUnaryArith(); +} + +template +bool BaselineCodeGen::emit_Lt() { + return emitCompare(); +} + +template +bool BaselineCodeGen::emit_Le() { + return emitCompare(); +} + +template +bool BaselineCodeGen::emit_Gt() { + return emitCompare(); +} + +template +bool BaselineCodeGen::emit_Ge() { + return emitCompare(); +} + +template +bool BaselineCodeGen::emit_Eq() { + return emitCompare(); +} + +template +bool BaselineCodeGen::emit_Ne() { + return emitCompare(); +} + +template +bool BaselineCodeGen::emitCompare() { + // Keep top JSStack value in R0 and R1. + frame.popRegsAndSync(2); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_StrictEq() { + return emitCompare(); +} + +template +bool BaselineCodeGen::emit_StrictNe() { + return emitCompare(); +} + +template +bool BaselineCodeGen::emit_Case() { + frame.popRegsAndSync(1); + + Label done; + masm.branchTestBooleanTruthy(/* branchIfTrue */ false, R0, &done); + { + // Pop the switch value if the case matches. + masm.addToStackPtr(Imm32(sizeof(Value))); + emitJump(); + } + masm.bind(&done); + return true; +} + +template +bool BaselineCodeGen::emit_Default() { + frame.pop(); + return emit_Goto(); +} + +template +bool BaselineCodeGen::emit_Lineno() { + return true; +} + +template +bool BaselineCodeGen::emit_NewArray() { + frame.syncStack(0); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0); + return true; +} + +static void MarkElementsNonPackedIfHoleValue(MacroAssembler& masm, + Register elements, + ValueOperand val) { + Label notHole; + masm.branchTestMagic(Assembler::NotEqual, val, ¬Hole); + { + Address elementsFlags(elements, ObjectElements::offsetOfFlags()); + masm.or32(Imm32(ObjectElements::NON_PACKED), elementsFlags); + } + masm.bind(¬Hole); +} + +template <> +bool BaselineInterpreterCodeGen::emit_InitElemArray() { + // Pop value into R0, keep the object on the stack. + frame.popRegsAndSync(1); + + // Load object in R2. + Register obj = R2.scratchReg(); + masm.unboxObject(frame.addressOfStackValue(-1), obj); + + // Load index in R1. + Register index = R1.scratchReg(); + LoadInt32Operand(masm, index); + + // Store the Value. No pre-barrier because this is an initialization. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), obj); + masm.storeValue(R0, BaseObjectElementIndex(obj, index)); + + // Bump initialized length. + Address initLength(obj, ObjectElements::offsetOfInitializedLength()); + masm.add32(Imm32(1), index); + masm.store32(index, initLength); + + // Mark elements as NON_PACKED if we stored the hole value. + MarkElementsNonPackedIfHoleValue(masm, obj, R0); + + // Post-barrier. + Label skipBarrier; + Register scratch = index; + masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, scratch, &skipBarrier); + { + masm.unboxObject(frame.addressOfStackValue(-1), obj); + masm.branchPtrInNurseryChunk(Assembler::Equal, obj, scratch, &skipBarrier); + MOZ_ASSERT(obj == R2.scratchReg(), "post barrier expects object in R2"); + masm.call(&postBarrierSlot_); + } + masm.bind(&skipBarrier); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_InitElemArray() { + // Pop value into R0, keep the object on the stack. + Maybe knownValue = frame.knownStackValue(-1); + frame.popRegsAndSync(1); + + // Load object in R2. + Register obj = R2.scratchReg(); + masm.unboxObject(frame.addressOfStackValue(-1), obj); + + uint32_t index = GET_UINT32(handler.pc()); + MOZ_ASSERT(index <= INT32_MAX, + "the bytecode emitter must fail to compile code that would " + "produce an index exceeding int32_t range"); + + // Store the Value. No pre-barrier because this is an initialization. + masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), obj); + masm.storeValue(R0, Address(obj, index * sizeof(Value))); + + // Bump initialized length. + Address initLength(obj, ObjectElements::offsetOfInitializedLength()); + masm.store32(Imm32(index + 1), initLength); + + // Mark elements as NON_PACKED if we stored the hole value. We know this + // statically except when debugger instrumentation is enabled because that + // forces a stack-sync (which discards constants and known types) for each op. + if (knownValue && knownValue->isMagic(JS_ELEMENTS_HOLE)) { + Address elementsFlags(obj, ObjectElements::offsetOfFlags()); + masm.or32(Imm32(ObjectElements::NON_PACKED), elementsFlags); + } else if (handler.compileDebugInstrumentation()) { + MarkElementsNonPackedIfHoleValue(masm, obj, R0); + } else { +#ifdef DEBUG + Label notHole; + masm.branchTestMagic(Assembler::NotEqual, R0, ¬Hole); + masm.assumeUnreachable("Unexpected hole value"); + masm.bind(¬Hole); +#endif + } + + // Post-barrier. + if (knownValue) { + MOZ_ASSERT(JS::GCPolicy::isTenured(*knownValue)); + } else { + Label skipBarrier; + Register scratch = R1.scratchReg(); + masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, scratch, + &skipBarrier); + { + masm.unboxObject(frame.addressOfStackValue(-1), obj); + masm.branchPtrInNurseryChunk(Assembler::Equal, obj, scratch, + &skipBarrier); + MOZ_ASSERT(obj == R2.scratchReg(), "post barrier expects object in R2"); + masm.call(&postBarrierSlot_); + } + masm.bind(&skipBarrier); + } + return true; +} + +template +bool BaselineCodeGen::emit_NewObject() { + return emitNewObject(); +} + +template +bool BaselineCodeGen::emit_NewInit() { + return emitNewObject(); +} + +template +bool BaselineCodeGen::emitNewObject() { + frame.syncStack(0); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_InitElem() { + // Store RHS in the scratch slot. + frame.storeStackValue(-1, frame.addressOfScratchValue(), R2); + frame.pop(); + + // Keep object and index in R0 and R1. + frame.popRegsAndSync(2); + + // Push the object to store the result of the IC. + frame.push(R0); + frame.syncStack(0); + + // Keep RHS on the stack. + frame.pushScratchValue(); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Pop the rhs, so that the object is on the top of the stack. + frame.pop(); + return true; +} + +template +bool BaselineCodeGen::emit_InitHiddenElem() { + return emit_InitElem(); +} + +template +bool BaselineCodeGen::emit_InitLockedElem() { + return emit_InitElem(); +} + +template +bool BaselineCodeGen::emit_MutateProto() { + // Keep values on the stack for the decompiler. + frame.syncStack(0); + + masm.unboxObject(frame.addressOfStackValue(-2), R0.scratchReg()); + masm.loadValue(frame.addressOfStackValue(-1), R1); + + prepareVMCall(); + + pushArg(R1); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, Handle, HandleValue); + if (!callVM()) { + return false; + } + + frame.pop(); + return true; +} + +template +bool BaselineCodeGen::emit_InitProp() { + // Load lhs in R0, rhs in R1. + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-2), R0); + masm.loadValue(frame.addressOfStackValue(-1), R1); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Leave the object on the stack. + frame.pop(); + return true; +} + +template +bool BaselineCodeGen::emit_InitLockedProp() { + return emit_InitProp(); +} + +template +bool BaselineCodeGen::emit_InitHiddenProp() { + return emit_InitProp(); +} + +template +bool BaselineCodeGen::emit_GetElem() { + // Keep top two stack values in R0 and R1. + frame.popRegsAndSync(2); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_GetElemSuper() { + // Store obj in the scratch slot. + frame.storeStackValue(-1, frame.addressOfScratchValue(), R2); + frame.pop(); + + // Keep receiver and index in R0 and R1. + frame.popRegsAndSync(2); + + // Keep obj on the stack. + frame.pushScratchValue(); + + if (!emitNextIC()) { + return false; + } + + frame.pop(); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_SetElem() { + // Store RHS in the scratch slot. + frame.storeStackValue(-1, frame.addressOfScratchValue(), R2); + frame.pop(); + + // Keep object and index in R0 and R1. + frame.popRegsAndSync(2); + + // Keep RHS on the stack. + frame.pushScratchValue(); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + return true; +} + +template +bool BaselineCodeGen::emit_StrictSetElem() { + return emit_SetElem(); +} + +template +bool BaselineCodeGen::emitSetElemSuper(bool strict) { + // Incoming stack is |receiver, propval, obj, rval|. We need to shuffle + // stack to leave rval when operation is complete. + + // Pop rval into R0, then load receiver into R1 and replace with rval. + frame.popRegsAndSync(1); + masm.loadValue(frame.addressOfStackValue(-3), R1); + masm.storeValue(R0, frame.addressOfStackValue(-3)); + + prepareVMCall(); + + pushArg(Imm32(strict)); + pushArg(R0); // rval + masm.loadValue(frame.addressOfStackValue(-2), R0); + pushArg(R0); // propval + pushArg(R1); // receiver + masm.loadValue(frame.addressOfStackValue(-1), R0); + pushArg(R0); // obj + + using Fn = bool (*)(JSContext*, HandleValue, HandleValue, HandleValue, + HandleValue, bool); + if (!callVM()) { + return false; + } + + frame.popn(2); + return true; +} + +template +bool BaselineCodeGen::emit_SetElemSuper() { + return emitSetElemSuper(/* strict = */ false); +} + +template +bool BaselineCodeGen::emit_StrictSetElemSuper() { + return emitSetElemSuper(/* strict = */ true); +} + +template +bool BaselineCodeGen::emitDelElem(bool strict) { + // Keep values on the stack for the decompiler. + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-2), R0); + masm.loadValue(frame.addressOfStackValue(-1), R1); + + prepareVMCall(); + + pushArg(R1); + pushArg(R0); + + using Fn = bool (*)(JSContext*, HandleValue, HandleValue, bool*); + if (strict) { + if (!callVM>()) { + return false; + } + } else { + if (!callVM>()) { + return false; + } + } + + masm.boxNonDouble(JSVAL_TYPE_BOOLEAN, ReturnReg, R1); + frame.popn(2); + frame.push(R1, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_DelElem() { + return emitDelElem(/* strict = */ false); +} + +template +bool BaselineCodeGen::emit_StrictDelElem() { + return emitDelElem(/* strict = */ true); +} + +template +bool BaselineCodeGen::emit_In() { + frame.popRegsAndSync(2); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_HasOwn() { + frame.popRegsAndSync(2); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_CheckPrivateField() { + // Keep key and val on the stack. + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-2), R0); + masm.loadValue(frame.addressOfStackValue(-1), R1); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_NewPrivateName() { + prepareVMCall(); + + pushScriptNameArg(R0.scratchReg(), R1.scratchReg()); + + using Fn = JS::Symbol* (*)(JSContext*, Handle); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_SYMBOL, ReturnReg, R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_GetGName() { + frame.syncStack(0); + + loadGlobalLexicalEnvironment(R0.scratchReg()); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::tryOptimizeBindGlobalName() { + JSScript* script = handler.script(); + MOZ_ASSERT(!script->hasNonSyntacticScope()); + + Rooted global(cx, &script->global()); + Rooted name(cx, script->getName(handler.pc())); + if (JSObject* binding = MaybeOptimizeBindGlobalName(cx, global, name)) { + frame.push(ObjectValue(*binding)); + return true; + } + return false; +} + +template <> +bool BaselineInterpreterCodeGen::tryOptimizeBindGlobalName() { + // Interpreter doesn't optimize simple BindGNames. + return false; +} + +template +bool BaselineCodeGen::emit_BindGName() { + if (tryOptimizeBindGlobalName()) { + return true; + } + + frame.syncStack(0); + loadGlobalLexicalEnvironment(R0.scratchReg()); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_BindVar() { + frame.syncStack(0); + masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg()); + + prepareVMCall(); + pushArg(R0.scratchReg()); + + using Fn = JSObject* (*)(JSContext*, JSObject*); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_SetProp() { + // Keep lhs in R0, rhs in R1. + frame.popRegsAndSync(2); + + // Keep RHS on the stack. + frame.push(R1); + frame.syncStack(0); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + return true; +} + +template +bool BaselineCodeGen::emit_StrictSetProp() { + return emit_SetProp(); +} + +template +bool BaselineCodeGen::emit_SetName() { + return emit_SetProp(); +} + +template +bool BaselineCodeGen::emit_StrictSetName() { + return emit_SetProp(); +} + +template +bool BaselineCodeGen::emit_SetGName() { + return emit_SetProp(); +} + +template +bool BaselineCodeGen::emit_StrictSetGName() { + return emit_SetProp(); +} + +template +bool BaselineCodeGen::emitSetPropSuper(bool strict) { + // Incoming stack is |receiver, obj, rval|. We need to shuffle stack to + // leave rval when operation is complete. + + // Pop rval into R0, then load receiver into R1 and replace with rval. + frame.popRegsAndSync(1); + masm.loadValue(frame.addressOfStackValue(-2), R1); + masm.storeValue(R0, frame.addressOfStackValue(-2)); + + prepareVMCall(); + + pushArg(Imm32(strict)); + pushArg(R0); // rval + pushScriptNameArg(R0.scratchReg(), R2.scratchReg()); + pushArg(R1); // receiver + masm.loadValue(frame.addressOfStackValue(-1), R0); + pushArg(R0); // obj + + using Fn = bool (*)(JSContext*, HandleValue, HandleValue, + Handle, HandleValue, bool); + if (!callVM()) { + return false; + } + + frame.pop(); + return true; +} + +template +bool BaselineCodeGen::emit_SetPropSuper() { + return emitSetPropSuper(/* strict = */ false); +} + +template +bool BaselineCodeGen::emit_StrictSetPropSuper() { + return emitSetPropSuper(/* strict = */ true); +} + +template +bool BaselineCodeGen::emit_GetProp() { + // Keep object in R0. + frame.popRegsAndSync(1); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_GetBoundName() { + return emit_GetProp(); +} + +template +bool BaselineCodeGen::emit_GetPropSuper() { + // Receiver -> R1, ObjectOrNull -> R0 + frame.popRegsAndSync(1); + masm.loadValue(frame.addressOfStackValue(-1), R1); + frame.pop(); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emitDelProp(bool strict) { + // Keep value on the stack for the decompiler. + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-1), R0); + + prepareVMCall(); + + pushScriptNameArg(R1.scratchReg(), R2.scratchReg()); + pushArg(R0); + + using Fn = bool (*)(JSContext*, HandleValue, Handle, bool*); + if (strict) { + if (!callVM>()) { + return false; + } + } else { + if (!callVM>()) { + return false; + } + } + + masm.boxNonDouble(JSVAL_TYPE_BOOLEAN, ReturnReg, R1); + frame.pop(); + frame.push(R1, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_DelProp() { + return emitDelProp(/* strict = */ false); +} + +template +bool BaselineCodeGen::emit_StrictDelProp() { + return emitDelProp(/* strict = */ true); +} + +template <> +void BaselineCompilerCodeGen::getEnvironmentCoordinateObject(Register reg) { + EnvironmentCoordinate ec(handler.pc()); + + masm.loadPtr(frame.addressOfEnvironmentChain(), reg); + for (unsigned i = ec.hops(); i; i--) { + masm.unboxObject( + Address(reg, EnvironmentObject::offsetOfEnclosingEnvironment()), reg); + } +} + +template <> +void BaselineInterpreterCodeGen::getEnvironmentCoordinateObject(Register reg) { + MOZ_CRASH("Shouldn't call this for interpreter"); +} + +template <> +Address BaselineCompilerCodeGen::getEnvironmentCoordinateAddressFromObject( + Register objReg, Register reg) { + EnvironmentCoordinate ec(handler.pc()); + + if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) { + return Address(objReg, NativeObject::getFixedSlotOffset(ec.slot())); + } + + uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec); + masm.loadPtr(Address(objReg, NativeObject::offsetOfSlots()), reg); + return Address(reg, slot * sizeof(Value)); +} + +template <> +Address BaselineInterpreterCodeGen::getEnvironmentCoordinateAddressFromObject( + Register objReg, Register reg) { + MOZ_CRASH("Shouldn't call this for interpreter"); +} + +template +Address BaselineCodeGen::getEnvironmentCoordinateAddress( + Register reg) { + getEnvironmentCoordinateObject(reg); + return getEnvironmentCoordinateAddressFromObject(reg, reg); +} + +// For a JOF_ENVCOORD op load the number of hops from the bytecode and skip this +// number of environment objects. +static void LoadAliasedVarEnv(MacroAssembler& masm, Register env, + Register scratch) { + static_assert(ENVCOORD_HOPS_LEN == 1, + "Code assumes number of hops is stored in uint8 operand"); + LoadUint8Operand(masm, scratch); + + Label top, done; + masm.branchTest32(Assembler::Zero, scratch, scratch, &done); + masm.bind(&top); + { + Address nextEnv(env, EnvironmentObject::offsetOfEnclosingEnvironment()); + masm.unboxObject(nextEnv, env); + masm.branchSub32(Assembler::NonZero, Imm32(1), scratch, &top); + } + masm.bind(&done); +} + +template <> +void BaselineCompilerCodeGen::emitGetAliasedVar(ValueOperand dest) { + frame.syncStack(0); + + Address address = getEnvironmentCoordinateAddress(R0.scratchReg()); + masm.loadValue(address, dest); +} + +template <> +void BaselineInterpreterCodeGen::emitGetAliasedVar(ValueOperand dest) { + Register env = R0.scratchReg(); + Register scratch = R1.scratchReg(); + + // Load the right environment object. + masm.loadPtr(frame.addressOfEnvironmentChain(), env); + LoadAliasedVarEnv(masm, env, scratch); + + // Load the slot index. + static_assert(ENVCOORD_SLOT_LEN == 3, + "Code assumes slot is stored in uint24 operand"); + LoadUint24Operand(masm, ENVCOORD_HOPS_LEN, scratch); + + // Load the Value from a fixed or dynamic slot. + // See EnvironmentObject::nonExtensibleIsFixedSlot. + Label isDynamic, done; + masm.branch32(Assembler::AboveOrEqual, scratch, + Imm32(NativeObject::MAX_FIXED_SLOTS), &isDynamic); + { + uint32_t offset = NativeObject::getFixedSlotOffset(0); + masm.loadValue(BaseValueIndex(env, scratch, offset), dest); + masm.jump(&done); + } + masm.bind(&isDynamic); + { + masm.loadPtr(Address(env, NativeObject::offsetOfSlots()), env); + + // Use an offset to subtract the number of fixed slots. + int32_t offset = -int32_t(NativeObject::MAX_FIXED_SLOTS * sizeof(Value)); + masm.loadValue(BaseValueIndex(env, scratch, offset), dest); + } + masm.bind(&done); +} + +template +bool BaselineCodeGen::emitGetAliasedDebugVar(ValueOperand dest) { + frame.syncStack(0); + Register env = R0.scratchReg(); + // Load the right environment object. + masm.loadPtr(frame.addressOfEnvironmentChain(), env); + + prepareVMCall(); + pushBytecodePCArg(); + pushArg(env); + + using Fn = + bool (*)(JSContext*, JSObject* env, jsbytecode*, MutableHandleValue); + return callVM(); +} + +template +bool BaselineCodeGen::emit_GetAliasedDebugVar() { + if (!emitGetAliasedDebugVar(R0)) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_GetAliasedVar() { + emitGetAliasedVar(R0); + + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_SetAliasedVar() { + // Keep rvalue in R0. + frame.popRegsAndSync(1); + Register objReg = R2.scratchReg(); + + getEnvironmentCoordinateObject(objReg); + Address address = + getEnvironmentCoordinateAddressFromObject(objReg, R1.scratchReg()); + masm.guardedCallPreBarrier(address, MIRType::Value); + masm.storeValue(R0, address); + frame.push(R0); + + // Only R0 is live at this point. + // Scope coordinate object is already in R2.scratchReg(). + Register temp = R1.scratchReg(); + + Label skipBarrier; + masm.branchPtrInNurseryChunk(Assembler::Equal, objReg, temp, &skipBarrier); + masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier); + + masm.call(&postBarrierSlot_); // Won't clobber R0 + + masm.bind(&skipBarrier); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_SetAliasedVar() { + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + MOZ_ASSERT(!regs.has(FramePointer)); + regs.take(R2); + if (HasInterpreterPCReg()) { + regs.take(InterpreterPCReg); + } + + Register env = regs.takeAny(); + Register scratch1 = regs.takeAny(); + Register scratch2 = regs.takeAny(); + Register scratch3 = regs.takeAny(); + + // Load the right environment object. + masm.loadPtr(frame.addressOfEnvironmentChain(), env); + LoadAliasedVarEnv(masm, env, scratch1); + + // Load the slot index. + static_assert(ENVCOORD_SLOT_LEN == 3, + "Code assumes slot is stored in uint24 operand"); + LoadUint24Operand(masm, ENVCOORD_HOPS_LEN, scratch1); + + // Store the RHS Value in R2. + masm.loadValue(frame.addressOfStackValue(-1), R2); + + // Load a pointer to the fixed or dynamic slot into scratch2. We want to call + // guardedCallPreBarrierAnyZone once to avoid code bloat. + + // See EnvironmentObject::nonExtensibleIsFixedSlot. + Label isDynamic, done; + masm.branch32(Assembler::AboveOrEqual, scratch1, + Imm32(NativeObject::MAX_FIXED_SLOTS), &isDynamic); + { + uint32_t offset = NativeObject::getFixedSlotOffset(0); + BaseValueIndex slotAddr(env, scratch1, offset); + masm.computeEffectiveAddress(slotAddr, scratch2); + masm.jump(&done); + } + masm.bind(&isDynamic); + { + masm.loadPtr(Address(env, NativeObject::offsetOfSlots()), scratch2); + + // Use an offset to subtract the number of fixed slots. + int32_t offset = -int32_t(NativeObject::MAX_FIXED_SLOTS * sizeof(Value)); + BaseValueIndex slotAddr(scratch2, scratch1, offset); + masm.computeEffectiveAddress(slotAddr, scratch2); + } + masm.bind(&done); + + // Pre-barrier and store. + Address slotAddr(scratch2, 0); + masm.guardedCallPreBarrierAnyZone(slotAddr, MIRType::Value, scratch3); + masm.storeValue(R2, slotAddr); + + // Post barrier. + Label skipBarrier; + masm.branchPtrInNurseryChunk(Assembler::Equal, env, scratch1, &skipBarrier); + masm.branchValueIsNurseryCell(Assembler::NotEqual, R2, scratch1, + &skipBarrier); + { + // Post barrier code expects the object in R2. + masm.movePtr(env, R2.scratchReg()); + masm.call(&postBarrierSlot_); + } + masm.bind(&skipBarrier); + return true; +} + +template +bool BaselineCodeGen::emit_GetName() { + frame.syncStack(0); + + masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg()); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_BindName() { + frame.syncStack(0); + masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg()); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_DelName() { + frame.syncStack(0); + masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg()); + + prepareVMCall(); + + pushArg(R0.scratchReg()); + pushScriptNameArg(R1.scratchReg(), R2.scratchReg()); + + using Fn = bool (*)(JSContext*, Handle, HandleObject, + MutableHandleValue); + if (!callVM()) { + return false; + } + + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_GetImport() { + JSScript* script = handler.script(); + ModuleEnvironmentObject* env = GetModuleEnvironmentForScript(script); + MOZ_ASSERT(env); + + jsid id = NameToId(script->getName(handler.pc())); + ModuleEnvironmentObject* targetEnv; + Maybe prop; + MOZ_ALWAYS_TRUE(env->lookupImport(id, &targetEnv, &prop)); + + frame.syncStack(0); + + uint32_t slot = prop->slot(); + Register scratch = R0.scratchReg(); + masm.movePtr(ImmGCPtr(targetEnv), scratch); + if (slot < targetEnv->numFixedSlots()) { + masm.loadValue(Address(scratch, NativeObject::getFixedSlotOffset(slot)), + R0); + } else { + masm.loadPtr(Address(scratch, NativeObject::offsetOfSlots()), scratch); + masm.loadValue( + Address(scratch, (slot - targetEnv->numFixedSlots()) * sizeof(Value)), + R0); + } + + // Imports are initialized by this point except in rare circumstances, so + // don't emit a check unless we have to. + if (targetEnv->getSlot(slot).isMagic(JS_UNINITIALIZED_LEXICAL)) { + if (!emitUninitializedLexicalCheck(R0)) { + return false; + } + } + + frame.push(R0); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_GetImport() { + frame.syncStack(0); + + masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg()); + + prepareVMCall(); + + pushBytecodePCArg(); + pushScriptArg(); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, HandleObject, HandleScript, jsbytecode*, + MutableHandleValue); + if (!callVM()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_GetIntrinsic() { + frame.syncStack(0); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_SetIntrinsic() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-1), R0); + + prepareVMCall(); + + pushArg(R0); + pushBytecodePCArg(); + pushScriptArg(); + + using Fn = bool (*)(JSContext*, JSScript*, jsbytecode*, HandleValue); + return callVM(); +} + +template +bool BaselineCodeGen::emit_GlobalOrEvalDeclInstantiation() { + frame.syncStack(0); + + prepareVMCall(); + + loadInt32LengthBytecodeOperand(R0.scratchReg()); + pushArg(R0.scratchReg()); + pushScriptArg(); + masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg()); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, HandleObject, HandleScript, GCThingIndex); + return callVM(); +} + +template +bool BaselineCodeGen::emitInitPropGetterSetter() { + // Keep values on the stack for the decompiler. + frame.syncStack(0); + + prepareVMCall(); + + masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg()); + masm.unboxObject(frame.addressOfStackValue(-2), R1.scratchReg()); + + pushArg(R0.scratchReg()); + pushScriptNameArg(R0.scratchReg(), R2.scratchReg()); + pushArg(R1.scratchReg()); + pushBytecodePCArg(); + + using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject, + Handle, HandleObject); + if (!callVM()) { + return false; + } + + frame.pop(); + return true; +} + +template +bool BaselineCodeGen::emit_InitPropGetter() { + return emitInitPropGetterSetter(); +} + +template +bool BaselineCodeGen::emit_InitHiddenPropGetter() { + return emitInitPropGetterSetter(); +} + +template +bool BaselineCodeGen::emit_InitPropSetter() { + return emitInitPropGetterSetter(); +} + +template +bool BaselineCodeGen::emit_InitHiddenPropSetter() { + return emitInitPropGetterSetter(); +} + +template +bool BaselineCodeGen::emitInitElemGetterSetter() { + // Load index and value in R0 and R1, but keep values on the stack for the + // decompiler. + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-2), R0); + masm.unboxObject(frame.addressOfStackValue(-1), R1.scratchReg()); + + prepareVMCall(); + + pushArg(R1.scratchReg()); + pushArg(R0); + masm.unboxObject(frame.addressOfStackValue(-3), R0.scratchReg()); + pushArg(R0.scratchReg()); + pushBytecodePCArg(); + + using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject, HandleValue, + HandleObject); + if (!callVM()) { + return false; + } + + frame.popn(2); + return true; +} + +template +bool BaselineCodeGen::emit_InitElemGetter() { + return emitInitElemGetterSetter(); +} + +template +bool BaselineCodeGen::emit_InitHiddenElemGetter() { + return emitInitElemGetterSetter(); +} + +template +bool BaselineCodeGen::emit_InitElemSetter() { + return emitInitElemGetterSetter(); +} + +template +bool BaselineCodeGen::emit_InitHiddenElemSetter() { + return emitInitElemGetterSetter(); +} + +template +bool BaselineCodeGen::emit_InitElemInc() { + // Keep the object and rhs on the stack. + frame.syncStack(0); + + // Load object in R0, index in R1. + masm.loadValue(frame.addressOfStackValue(-3), R0); + masm.loadValue(frame.addressOfStackValue(-2), R1); + + // Call IC. + if (!emitNextIC()) { + return false; + } + + // Pop the rhs + frame.pop(); + + // Increment index + Address indexAddr = frame.addressOfStackValue(-1); +#ifdef DEBUG + Label isInt32; + masm.branchTestInt32(Assembler::Equal, indexAddr, &isInt32); + masm.assumeUnreachable("INITELEM_INC index must be Int32"); + masm.bind(&isInt32); +#endif + masm.incrementInt32Value(indexAddr); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_GetLocal() { + frame.pushLocal(GET_LOCALNO(handler.pc())); + return true; +} + +static BaseValueIndex ComputeAddressOfLocal(MacroAssembler& masm, + Register indexScratch) { + // Locals are stored in memory at a negative offset from the frame pointer. We + // negate the index first to effectively subtract it. + masm.negPtr(indexScratch); + return BaseValueIndex(FramePointer, indexScratch, + BaselineFrame::reverseOffsetOfLocal(0)); +} + +template <> +bool BaselineInterpreterCodeGen::emit_GetLocal() { + Register scratch = R0.scratchReg(); + LoadUint24Operand(masm, 0, scratch); + BaseValueIndex addr = ComputeAddressOfLocal(masm, scratch); + masm.loadValue(addr, R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_SetLocal() { + // Ensure no other StackValue refers to the old value, for instance i + (i = + // 3). This also allows us to use R0 as scratch below. + frame.syncStack(1); + + uint32_t local = GET_LOCALNO(handler.pc()); + frame.storeStackValue(-1, frame.addressOfLocal(local), R0); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_SetLocal() { + Register scratch = R0.scratchReg(); + LoadUint24Operand(masm, 0, scratch); + BaseValueIndex addr = ComputeAddressOfLocal(masm, scratch); + masm.loadValue(frame.addressOfStackValue(-1), R1); + masm.storeValue(R1, addr); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emitFormalArgAccess(JSOp op) { + MOZ_ASSERT(op == JSOp::GetArg || op == JSOp::SetArg); + + uint32_t arg = GET_ARGNO(handler.pc()); + + // Fast path: the script does not use |arguments| or formals don't + // alias the arguments object. + if (!handler.script()->argsObjAliasesFormals()) { + if (op == JSOp::GetArg) { + frame.pushArg(arg); + } else { + // See the comment in emit_SetLocal. + frame.syncStack(1); + frame.storeStackValue(-1, frame.addressOfArg(arg), R0); + } + + return true; + } + + // Sync so that we can use R0. + frame.syncStack(0); + + // Load the arguments object data vector. + Register reg = R2.scratchReg(); + masm.loadPtr(frame.addressOfArgsObj(), reg); + masm.loadPrivate(Address(reg, ArgumentsObject::getDataSlotOffset()), reg); + + // Load/store the argument. + Address argAddr(reg, ArgumentsData::offsetOfArgs() + arg * sizeof(Value)); + if (op == JSOp::GetArg) { + masm.loadValue(argAddr, R0); + frame.push(R0); + } else { + Register temp = R1.scratchReg(); + masm.guardedCallPreBarrierAnyZone(argAddr, MIRType::Value, temp); + masm.loadValue(frame.addressOfStackValue(-1), R0); + masm.storeValue(R0, argAddr); + + MOZ_ASSERT(frame.numUnsyncedSlots() == 0); + + // Reload the arguments object. + Register reg = R2.scratchReg(); + masm.loadPtr(frame.addressOfArgsObj(), reg); + + Label skipBarrier; + + masm.branchPtrInNurseryChunk(Assembler::Equal, reg, temp, &skipBarrier); + masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier); + + masm.call(&postBarrierSlot_); + + masm.bind(&skipBarrier); + } + + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emitFormalArgAccess(JSOp op) { + MOZ_ASSERT(op == JSOp::GetArg || op == JSOp::SetArg); + + // Load the index. + Register argReg = R1.scratchReg(); + LoadUint16Operand(masm, argReg); + + // If the frame has no arguments object, this must be an unaliased access. + Label isUnaliased, done; + masm.branchTest32(Assembler::Zero, frame.addressOfFlags(), + Imm32(BaselineFrame::HAS_ARGS_OBJ), &isUnaliased); + { + Register reg = R2.scratchReg(); + + // If it's an unmapped arguments object, this is an unaliased access. + loadScript(reg); + masm.branchTest32( + Assembler::Zero, Address(reg, JSScript::offsetOfImmutableFlags()), + Imm32(uint32_t(JSScript::ImmutableFlags::HasMappedArgsObj)), + &isUnaliased); + + // Load the arguments object data vector. + masm.loadPtr(frame.addressOfArgsObj(), reg); + masm.loadPrivate(Address(reg, ArgumentsObject::getDataSlotOffset()), reg); + + // Load/store the argument. + BaseValueIndex argAddr(reg, argReg, ArgumentsData::offsetOfArgs()); + if (op == JSOp::GetArg) { + masm.loadValue(argAddr, R0); + frame.push(R0); + } else { + masm.guardedCallPreBarrierAnyZone(argAddr, MIRType::Value, + R0.scratchReg()); + masm.loadValue(frame.addressOfStackValue(-1), R0); + masm.storeValue(R0, argAddr); + + // Reload the arguments object. + masm.loadPtr(frame.addressOfArgsObj(), reg); + + Register temp = R1.scratchReg(); + masm.branchPtrInNurseryChunk(Assembler::Equal, reg, temp, &done); + masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &done); + + masm.call(&postBarrierSlot_); + } + masm.jump(&done); + } + masm.bind(&isUnaliased); + { + BaseValueIndex addr(FramePointer, argReg, + JitFrameLayout::offsetOfActualArgs()); + if (op == JSOp::GetArg) { + masm.loadValue(addr, R0); + frame.push(R0); + } else { + masm.loadValue(frame.addressOfStackValue(-1), R0); + masm.storeValue(R0, addr); + } + } + + masm.bind(&done); + return true; +} + +template +bool BaselineCodeGen::emit_GetArg() { + return emitFormalArgAccess(JSOp::GetArg); +} + +template +bool BaselineCodeGen::emit_SetArg() { + return emitFormalArgAccess(JSOp::SetArg); +} + +template <> +bool BaselineInterpreterCodeGen::emit_GetFrameArg() { + frame.syncStack(0); + + Register argReg = R1.scratchReg(); + LoadUint16Operand(masm, argReg); + + BaseValueIndex addr(FramePointer, argReg, + JitFrameLayout::offsetOfActualArgs()); + masm.loadValue(addr, R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_GetFrameArg() { + uint32_t arg = GET_ARGNO(handler.pc()); + frame.pushArg(arg); + return true; +} + +template +bool BaselineCodeGen::emit_ArgumentsLength() { + frame.syncStack(0); + + masm.loadNumActualArgs(FramePointer, R0.scratchReg()); + masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0); + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_GetActualArg() { + frame.popRegsAndSync(1); + +#ifdef DEBUG + { + Label ok; + masm.branchTestInt32(Assembler::Equal, R0, &ok); + masm.assumeUnreachable("GetActualArg unexpected type"); + masm.bind(&ok); + } +#endif + + Register index = R0.scratchReg(); + masm.unboxInt32(R0, index); + +#ifdef DEBUG + { + Label ok; + masm.loadNumActualArgs(FramePointer, R1.scratchReg()); + masm.branch32(Assembler::Above, R1.scratchReg(), index, &ok); + masm.assumeUnreachable("GetActualArg invalid index"); + masm.bind(&ok); + } +#endif + + BaseValueIndex addr(FramePointer, index, + JitFrameLayout::offsetOfActualArgs()); + masm.loadValue(addr, R0); + frame.push(R0); + return true; +} + +template <> +void BaselineCompilerCodeGen::loadNumFormalArguments(Register dest) { + masm.move32(Imm32(handler.function()->nargs()), dest); +} + +template <> +void BaselineInterpreterCodeGen::loadNumFormalArguments(Register dest) { + masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(), dest); + masm.loadFunctionArgCount(dest, dest); +} + +template +bool BaselineCodeGen::emit_NewTarget() { + MOZ_ASSERT_IF(handler.maybeFunction(), !handler.maybeFunction()->isArrow()); + + frame.syncStack(0); + +#ifdef DEBUG + Register scratch1 = R0.scratchReg(); + Register scratch2 = R1.scratchReg(); + + Label isFunction; + masm.loadPtr(frame.addressOfCalleeToken(), scratch1); + masm.branchTestPtr(Assembler::Zero, scratch1, Imm32(CalleeTokenScriptBit), + &isFunction); + masm.assumeUnreachable("Unexpected non-function script"); + masm.bind(&isFunction); + + Label notArrow; + masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), scratch1); + masm.branchFunctionKind(Assembler::NotEqual, + FunctionFlags::FunctionKind::Arrow, scratch1, + scratch2, ¬Arrow); + masm.assumeUnreachable("Unexpected arrow function"); + masm.bind(¬Arrow); +#endif + + // if (isConstructing()) push(argv[Max(numActualArgs, numFormalArgs)]) + Label notConstructing, done; + masm.branchTestPtr(Assembler::Zero, frame.addressOfCalleeToken(), + Imm32(CalleeToken_FunctionConstructing), ¬Constructing); + { + Register argvLen = R0.scratchReg(); + Register nformals = R1.scratchReg(); + masm.loadNumActualArgs(FramePointer, argvLen); + + // If argvLen < nformals, set argvlen := nformals. + loadNumFormalArguments(nformals); + masm.cmp32Move32(Assembler::Below, argvLen, nformals, nformals, argvLen); + + BaseValueIndex newTarget(FramePointer, argvLen, + JitFrameLayout::offsetOfActualArgs()); + masm.loadValue(newTarget, R0); + masm.jump(&done); + } + // else push(undefined) + masm.bind(¬Constructing); + masm.moveValue(UndefinedValue(), R0); + + masm.bind(&done); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_ThrowSetConst() { + prepareVMCall(); + pushArg(Imm32(JSMSG_BAD_CONST_ASSIGN)); + + using Fn = bool (*)(JSContext*, unsigned); + return callVM(); +} + +template +bool BaselineCodeGen::emitUninitializedLexicalCheck( + const ValueOperand& val) { + Label done; + masm.branchTestMagicValue(Assembler::NotEqual, val, JS_UNINITIALIZED_LEXICAL, + &done); + + prepareVMCall(); + pushArg(Imm32(JSMSG_UNINITIALIZED_LEXICAL)); + + using Fn = bool (*)(JSContext*, unsigned); + if (!callVM()) { + return false; + } + + masm.bind(&done); + return true; +} + +template +bool BaselineCodeGen::emit_CheckLexical() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-1), R0); + return emitUninitializedLexicalCheck(R0); +} + +template +bool BaselineCodeGen::emit_CheckAliasedLexical() { + return emit_CheckLexical(); +} + +template +bool BaselineCodeGen::emit_InitLexical() { + return emit_SetLocal(); +} + +template +bool BaselineCodeGen::emit_InitGLexical() { + frame.popRegsAndSync(1); + pushGlobalLexicalEnvironmentValue(R1); + frame.push(R0); + return emit_SetProp(); +} + +template +bool BaselineCodeGen::emit_InitAliasedLexical() { + return emit_SetAliasedVar(); +} + +template +bool BaselineCodeGen::emit_Uninitialized() { + frame.push(MagicValue(JS_UNINITIALIZED_LEXICAL)); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emitCall(JSOp op) { + MOZ_ASSERT(IsInvokeOp(op)); + + frame.syncStack(0); + + uint32_t argc = GET_ARGC(handler.pc()); + masm.move32(Imm32(argc), R0.scratchReg()); + + // Call IC + if (!emitNextIC()) { + return false; + } + + // Update FrameInfo. + bool construct = IsConstructOp(op); + frame.popn(2 + argc + construct); + frame.push(R0); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emitCall(JSOp op) { + MOZ_ASSERT(IsInvokeOp(op)); + + // The IC expects argc in R0. + LoadUint16Operand(masm, R0.scratchReg()); + if (!emitNextIC()) { + return false; + } + + // Pop the arguments. We have to reload pc/argc because the IC clobbers them. + // The return value is in R0 so we can't use that. + Register scratch = R1.scratchReg(); + uint32_t extraValuesToPop = IsConstructOp(op) ? 3 : 2; + Register spReg = AsRegister(masm.getStackPointer()); + LoadUint16Operand(masm, scratch); + masm.computeEffectiveAddress( + BaseValueIndex(spReg, scratch, extraValuesToPop * sizeof(Value)), spReg); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emitSpreadCall(JSOp op) { + MOZ_ASSERT(IsInvokeOp(op)); + + frame.syncStack(0); + masm.move32(Imm32(1), R0.scratchReg()); + + // Call IC + if (!emitNextIC()) { + return false; + } + + // Update FrameInfo. + bool construct = op == JSOp::SpreadNew || op == JSOp::SpreadSuperCall; + frame.popn(3 + construct); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_Call() { + return emitCall(JSOp::Call); +} + +template +bool BaselineCodeGen::emit_CallContent() { + return emitCall(JSOp::CallContent); +} + +template +bool BaselineCodeGen::emit_CallIgnoresRv() { + return emitCall(JSOp::CallIgnoresRv); +} + +template +bool BaselineCodeGen::emit_CallIter() { + return emitCall(JSOp::CallIter); +} + +template +bool BaselineCodeGen::emit_CallContentIter() { + return emitCall(JSOp::CallContentIter); +} + +template +bool BaselineCodeGen::emit_New() { + return emitCall(JSOp::New); +} + +template +bool BaselineCodeGen::emit_NewContent() { + return emitCall(JSOp::NewContent); +} + +template +bool BaselineCodeGen::emit_SuperCall() { + return emitCall(JSOp::SuperCall); +} + +template +bool BaselineCodeGen::emit_Eval() { + return emitCall(JSOp::Eval); +} + +template +bool BaselineCodeGen::emit_StrictEval() { + return emitCall(JSOp::StrictEval); +} + +template +bool BaselineCodeGen::emit_SpreadCall() { + return emitSpreadCall(JSOp::SpreadCall); +} + +template +bool BaselineCodeGen::emit_SpreadNew() { + return emitSpreadCall(JSOp::SpreadNew); +} + +template +bool BaselineCodeGen::emit_SpreadSuperCall() { + return emitSpreadCall(JSOp::SpreadSuperCall); +} + +template +bool BaselineCodeGen::emit_SpreadEval() { + return emitSpreadCall(JSOp::SpreadEval); +} + +template +bool BaselineCodeGen::emit_StrictSpreadEval() { + return emitSpreadCall(JSOp::StrictSpreadEval); +} + +template +bool BaselineCodeGen::emit_OptimizeSpreadCall() { + frame.popRegsAndSync(1); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_ImplicitThis() { + frame.syncStack(0); + masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg()); + + prepareVMCall(); + + pushScriptNameArg(R1.scratchReg(), R2.scratchReg()); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, HandleObject, Handle, + MutableHandleValue); + if (!callVM()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_Instanceof() { + frame.popRegsAndSync(2); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_Typeof() { + frame.popRegsAndSync(1); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_TypeofExpr() { + return emit_Typeof(); +} + +template +bool BaselineCodeGen::emit_ThrowMsg() { + prepareVMCall(); + pushUint8BytecodeOperandArg(R2.scratchReg()); + + using Fn = bool (*)(JSContext*, const unsigned); + return callVM(); +} + +template +bool BaselineCodeGen::emit_Throw() { + // Keep value to throw in R0. + frame.popRegsAndSync(1); + + prepareVMCall(); + pushArg(R0); + + using Fn = bool (*)(JSContext*, HandleValue); + return callVM(); +} + +template +bool BaselineCodeGen::emit_Try() { + return true; +} + +template +bool BaselineCodeGen::emit_Finally() { + // To match the interpreter, emit an interrupt check at the start of the + // finally block. + return emitInterruptCheck(); +} + +static void LoadBaselineScriptResumeEntries(MacroAssembler& masm, + JSScript* script, Register dest, + Register scratch) { + MOZ_ASSERT(dest != scratch); + + masm.movePtr(ImmPtr(script->jitScript()), dest); + masm.loadPtr(Address(dest, JitScript::offsetOfBaselineScript()), dest); + masm.load32(Address(dest, BaselineScript::offsetOfResumeEntriesOffset()), + scratch); + masm.addPtr(scratch, dest); +} + +template +void BaselineCodeGen::emitInterpJumpToResumeEntry(Register script, + Register resumeIndex, + Register scratch) { + // Load JSScript::immutableScriptData() into |script|. + masm.loadPtr(Address(script, JSScript::offsetOfSharedData()), script); + masm.loadPtr(Address(script, SharedImmutableScriptData::offsetOfISD()), + script); + + // Load the resume pcOffset in |resumeIndex|. + masm.load32( + Address(script, ImmutableScriptData::offsetOfResumeOffsetsOffset()), + scratch); + masm.computeEffectiveAddress(BaseIndex(scratch, resumeIndex, TimesFour), + scratch); + masm.load32(BaseIndex(script, scratch, TimesOne), resumeIndex); + + // Add resume offset to PC, jump to it. + masm.computeEffectiveAddress(BaseIndex(script, resumeIndex, TimesOne, + ImmutableScriptData::offsetOfCode()), + script); + Address pcAddr(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC()); + masm.storePtr(script, pcAddr); + emitJumpToInterpretOpLabel(); +} + +template <> +void BaselineCompilerCodeGen::jumpToResumeEntry(Register resumeIndex, + Register scratch1, + Register scratch2) { + LoadBaselineScriptResumeEntries(masm, handler.script(), scratch1, scratch2); + masm.loadPtr( + BaseIndex(scratch1, resumeIndex, ScaleFromElemWidth(sizeof(uintptr_t))), + scratch1); + masm.jump(scratch1); +} + +template <> +void BaselineInterpreterCodeGen::jumpToResumeEntry(Register resumeIndex, + Register scratch1, + Register scratch2) { + loadScript(scratch1); + emitInterpJumpToResumeEntry(scratch1, resumeIndex, scratch2); +} + +template <> +template +[[nodiscard]] bool BaselineCompilerCodeGen::emitDebugInstrumentation( + const F1& ifDebuggee, const Maybe& ifNotDebuggee) { + // The JIT calls either ifDebuggee or (if present) ifNotDebuggee, because it + // knows statically whether we're compiling with debug instrumentation. + + if (handler.compileDebugInstrumentation()) { + return ifDebuggee(); + } + + if (ifNotDebuggee) { + return (*ifNotDebuggee)(); + } + + return true; +} + +template <> +template +[[nodiscard]] bool BaselineInterpreterCodeGen::emitDebugInstrumentation( + const F1& ifDebuggee, const Maybe& ifNotDebuggee) { + // The interpreter emits both ifDebuggee and (if present) ifNotDebuggee + // paths, with a toggled jump followed by a branch on the frame's DEBUGGEE + // flag. + + Label isNotDebuggee, done; + + CodeOffset toggleOffset = masm.toggledJump(&isNotDebuggee); + if (!handler.addDebugInstrumentationOffset(cx, toggleOffset)) { + return false; + } + + masm.branchTest32(Assembler::Zero, frame.addressOfFlags(), + Imm32(BaselineFrame::DEBUGGEE), &isNotDebuggee); + + if (!ifDebuggee()) { + return false; + } + + if (ifNotDebuggee) { + masm.jump(&done); + } + + masm.bind(&isNotDebuggee); + + if (ifNotDebuggee && !(*ifNotDebuggee)()) { + return false; + } + + masm.bind(&done); + return true; +} + +template +bool BaselineCodeGen::emit_PushLexicalEnv() { + // Call a stub to push the block on the block chain. + prepareVMCall(); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + + pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(), + R2.scratchReg()); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, Handle); + return callVM(); +} + +template +bool BaselineCodeGen::emit_PushClassBodyEnv() { + prepareVMCall(); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + + pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(), + R2.scratchReg()); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, Handle); + return callVM(); +} + +template +bool BaselineCodeGen::emit_PopLexicalEnv() { + frame.syncStack(0); + + Register scratch1 = R0.scratchReg(); + + auto ifDebuggee = [this, scratch1]() { + masm.loadBaselineFramePtr(FramePointer, scratch1); + + prepareVMCall(); + pushBytecodePCArg(); + pushArg(scratch1); + + using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*); + return callVM(); + }; + auto ifNotDebuggee = [this, scratch1]() { + Register scratch2 = R1.scratchReg(); + masm.loadPtr(frame.addressOfEnvironmentChain(), scratch1); + masm.debugAssertObjectHasClass(scratch1, scratch2, + &LexicalEnvironmentObject::class_); + Address enclosingAddr(scratch1, + EnvironmentObject::offsetOfEnclosingEnvironment()); + masm.unboxObject(enclosingAddr, scratch1); + masm.storePtr(scratch1, frame.addressOfEnvironmentChain()); + return true; + }; + return emitDebugInstrumentation(ifDebuggee, mozilla::Some(ifNotDebuggee)); +} + +template +bool BaselineCodeGen::emit_FreshenLexicalEnv() { + frame.syncStack(0); + + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + + auto ifDebuggee = [this]() { + prepareVMCall(); + pushBytecodePCArg(); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*); + return callVM(); + }; + auto ifNotDebuggee = [this]() { + prepareVMCall(); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*); + return callVM(); + }; + return emitDebugInstrumentation(ifDebuggee, mozilla::Some(ifNotDebuggee)); +} + +template +bool BaselineCodeGen::emit_RecreateLexicalEnv() { + frame.syncStack(0); + + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + + auto ifDebuggee = [this]() { + prepareVMCall(); + pushBytecodePCArg(); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*); + return callVM(); + }; + auto ifNotDebuggee = [this]() { + prepareVMCall(); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*); + return callVM(); + }; + return emitDebugInstrumentation(ifDebuggee, mozilla::Some(ifNotDebuggee)); +} + +template +bool BaselineCodeGen::emit_DebugLeaveLexicalEnv() { + auto ifDebuggee = [this]() { + prepareVMCall(); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + pushBytecodePCArg(); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*); + return callVM(); + }; + return emitDebugInstrumentation(ifDebuggee); +} + +template +bool BaselineCodeGen::emit_PushVarEnv() { + prepareVMCall(); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(), + R2.scratchReg()); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, Handle); + return callVM(); +} + +template +bool BaselineCodeGen::emit_EnterWith() { + // Pop "with" object to R0. + frame.popRegsAndSync(1); + + // Call a stub to push the object onto the environment chain. + prepareVMCall(); + + pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(), + R2.scratchReg()); + pushArg(R0); + masm.loadBaselineFramePtr(FramePointer, R1.scratchReg()); + pushArg(R1.scratchReg()); + + using Fn = + bool (*)(JSContext*, BaselineFrame*, HandleValue, Handle); + return callVM(); +} + +template +bool BaselineCodeGen::emit_LeaveWith() { + // Call a stub to pop the with object from the environment chain. + prepareVMCall(); + + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*); + return callVM(); +} + +template +bool BaselineCodeGen::emit_Exception() { + prepareVMCall(); + + using Fn = bool (*)(JSContext*, MutableHandleValue); + if (!callVM()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_Debugger() { + prepareVMCall(); + + frame.assertSyncedStack(); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*); + if (!callVM()) { + return false; + } + + return true; +} + +template +bool BaselineCodeGen::emitDebugEpilogue() { + auto ifDebuggee = [this]() { + // Move return value into the frame's rval slot. + masm.storeValue(JSReturnOperand, frame.addressOfReturnValue()); + masm.or32(Imm32(BaselineFrame::HAS_RVAL), frame.addressOfFlags()); + + // Load BaselineFrame pointer in R0. + frame.syncStack(0); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + + prepareVMCall(); + pushBytecodePCArg(); + pushArg(R0.scratchReg()); + + const RetAddrEntry::Kind kind = RetAddrEntry::Kind::DebugEpilogue; + + using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*); + if (!callVM(kind)) { + return false; + } + + masm.loadValue(frame.addressOfReturnValue(), JSReturnOperand); + return true; + }; + return emitDebugInstrumentation(ifDebuggee); +} + +template +bool BaselineCodeGen::emitReturn() { + if (handler.shouldEmitDebugEpilogueAtReturnOp()) { + if (!emitDebugEpilogue()) { + return false; + } + } + + // Only emit the jump if this JSOp::RetRval is not the last instruction. + // Not needed for last instruction, because last instruction flows + // into return label. + if (!handler.isDefinitelyLastOp()) { + masm.jump(&return_); + } + + return true; +} + +template +bool BaselineCodeGen::emit_Return() { + frame.assertStackDepth(1); + + frame.popValue(JSReturnOperand); + return emitReturn(); +} + +template +void BaselineCodeGen::emitLoadReturnValue(ValueOperand val) { + Label done, noRval; + masm.branchTest32(Assembler::Zero, frame.addressOfFlags(), + Imm32(BaselineFrame::HAS_RVAL), &noRval); + masm.loadValue(frame.addressOfReturnValue(), val); + masm.jump(&done); + + masm.bind(&noRval); + masm.moveValue(UndefinedValue(), val); + + masm.bind(&done); +} + +template +bool BaselineCodeGen::emit_RetRval() { + frame.assertStackDepth(0); + + masm.moveValue(UndefinedValue(), JSReturnOperand); + + if (!handler.maybeScript() || !handler.maybeScript()->noScriptRval()) { + // Return the value in the return value slot, if any. + Label done; + Address flags = frame.addressOfFlags(); + masm.branchTest32(Assembler::Zero, flags, Imm32(BaselineFrame::HAS_RVAL), + &done); + masm.loadValue(frame.addressOfReturnValue(), JSReturnOperand); + masm.bind(&done); + } + + return emitReturn(); +} + +template +bool BaselineCodeGen::emit_ToPropertyKey() { + frame.popRegsAndSync(1); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_ToAsyncIter() { + frame.syncStack(0); + masm.unboxObject(frame.addressOfStackValue(-2), R0.scratchReg()); + masm.loadValue(frame.addressOfStackValue(-1), R1); + + prepareVMCall(); + pushArg(R1); + pushArg(R0.scratchReg()); + + using Fn = JSObject* (*)(JSContext*, HandleObject, HandleValue); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.popn(2); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_CanSkipAwait() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-1), R0); + + prepareVMCall(); + pushArg(R0); + + using Fn = bool (*)(JSContext*, HandleValue, bool* canSkip); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, R0); + frame.push(R0, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_MaybeExtractAwaitValue() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-2), R0); + + masm.unboxBoolean(frame.addressOfStackValue(-1), R1.scratchReg()); + + Label cantExtract; + masm.branchIfFalseBool(R1.scratchReg(), &cantExtract); + + prepareVMCall(); + pushArg(R0); + + using Fn = bool (*)(JSContext*, HandleValue, MutableHandleValue); + if (!callVM()) { + return false; + } + + masm.storeValue(R0, frame.addressOfStackValue(-2)); + masm.bind(&cantExtract); + + return true; +} + +template +bool BaselineCodeGen::emit_AsyncAwait() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-2), R1); + masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg()); + + prepareVMCall(); + pushArg(R1); + pushArg(R0.scratchReg()); + + using Fn = JSObject* (*)(JSContext*, Handle, + HandleValue); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.popn(2); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_AsyncResolve() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-2), R1); + masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg()); + + prepareVMCall(); + pushUint8BytecodeOperandArg(R2.scratchReg()); + pushArg(R1); + pushArg(R0.scratchReg()); + + using Fn = JSObject* (*)(JSContext*, Handle, + HandleValue, AsyncFunctionResolveKind); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.popn(2); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_CheckObjCoercible() { + frame.syncStack(0); + masm.loadValue(frame.addressOfStackValue(-1), R0); + + Label fail, done; + + masm.branchTestUndefined(Assembler::Equal, R0, &fail); + masm.branchTestNull(Assembler::NotEqual, R0, &done); + + masm.bind(&fail); + prepareVMCall(); + + pushArg(R0); + + using Fn = bool (*)(JSContext*, HandleValue); + if (!callVM()) { + return false; + } + + masm.bind(&done); + return true; +} + +template +bool BaselineCodeGen::emit_ToString() { + // Keep top stack value in R0. + frame.popRegsAndSync(1); + + // Inline path for string. + Label done; + masm.branchTestString(Assembler::Equal, R0, &done); + + prepareVMCall(); + + pushArg(R0); + + // Call ToStringSlow which doesn't handle string inputs. + using Fn = JSString* (*)(JSContext*, HandleValue); + if (!callVM>()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_STRING, ReturnReg, R0); + + masm.bind(&done); + frame.push(R0); + return true; +} + +static constexpr uint32_t TableSwitchOpLowOffset = 1 * JUMP_OFFSET_LEN; +static constexpr uint32_t TableSwitchOpHighOffset = 2 * JUMP_OFFSET_LEN; +static constexpr uint32_t TableSwitchOpFirstResumeIndexOffset = + 3 * JUMP_OFFSET_LEN; + +template <> +void BaselineCompilerCodeGen::emitGetTableSwitchIndex(ValueOperand val, + Register dest, + Register scratch1, + Register scratch2) { + jsbytecode* pc = handler.pc(); + jsbytecode* defaultpc = pc + GET_JUMP_OFFSET(pc); + Label* defaultLabel = handler.labelOf(defaultpc); + + int32_t low = GET_JUMP_OFFSET(pc + TableSwitchOpLowOffset); + int32_t high = GET_JUMP_OFFSET(pc + TableSwitchOpHighOffset); + int32_t length = high - low + 1; + + // Jump to the 'default' pc if not int32 (tableswitch is only used when + // all cases are int32). + masm.branchTestInt32(Assembler::NotEqual, val, defaultLabel); + masm.unboxInt32(val, dest); + + // Subtract 'low'. Bounds check. + if (low != 0) { + masm.sub32(Imm32(low), dest); + } + masm.branch32(Assembler::AboveOrEqual, dest, Imm32(length), defaultLabel); +} + +template <> +void BaselineInterpreterCodeGen::emitGetTableSwitchIndex(ValueOperand val, + Register dest, + Register scratch1, + Register scratch2) { + // Jump to the 'default' pc if not int32 (tableswitch is only used when + // all cases are int32). + Label done, jumpToDefault; + masm.branchTestInt32(Assembler::NotEqual, val, &jumpToDefault); + masm.unboxInt32(val, dest); + + Register pcReg = LoadBytecodePC(masm, scratch1); + Address lowAddr(pcReg, sizeof(jsbytecode) + TableSwitchOpLowOffset); + Address highAddr(pcReg, sizeof(jsbytecode) + TableSwitchOpHighOffset); + + // Jump to default if val > high. + masm.branch32(Assembler::LessThan, highAddr, dest, &jumpToDefault); + + // Jump to default if val < low. + masm.load32(lowAddr, scratch2); + masm.branch32(Assembler::GreaterThan, scratch2, dest, &jumpToDefault); + + // index := val - low. + masm.sub32(scratch2, dest); + masm.jump(&done); + + masm.bind(&jumpToDefault); + emitJump(); + + masm.bind(&done); +} + +template <> +void BaselineCompilerCodeGen::emitTableSwitchJump(Register key, + Register scratch1, + Register scratch2) { + // Jump to resumeEntries[firstResumeIndex + key]. + + // Note: BytecodeEmitter::allocateResumeIndex static_asserts + // |firstResumeIndex * sizeof(uintptr_t)| fits in int32_t. + uint32_t firstResumeIndex = + GET_RESUMEINDEX(handler.pc() + TableSwitchOpFirstResumeIndexOffset); + LoadBaselineScriptResumeEntries(masm, handler.script(), scratch1, scratch2); + masm.loadPtr(BaseIndex(scratch1, key, ScaleFromElemWidth(sizeof(uintptr_t)), + firstResumeIndex * sizeof(uintptr_t)), + scratch1); + masm.jump(scratch1); +} + +template <> +void BaselineInterpreterCodeGen::emitTableSwitchJump(Register key, + Register scratch1, + Register scratch2) { + // Load the op's firstResumeIndex in scratch1. + LoadUint24Operand(masm, TableSwitchOpFirstResumeIndexOffset, scratch1); + + masm.add32(key, scratch1); + jumpToResumeEntry(scratch1, key, scratch2); +} + +template +bool BaselineCodeGen::emit_TableSwitch() { + frame.popRegsAndSync(1); + + Register key = R0.scratchReg(); + Register scratch1 = R1.scratchReg(); + Register scratch2 = R2.scratchReg(); + + // Call a stub to convert R0 from double to int32 if needed. + // Note: this stub may clobber scratch1. + masm.call(cx->runtime()->jitRuntime()->getDoubleToInt32ValueStub()); + + // Load the index in the jump table in |key|, or branch to default pc if not + // int32 or out-of-range. + emitGetTableSwitchIndex(R0, key, scratch1, scratch2); + + // Jump to the target pc. + emitTableSwitchJump(key, scratch1, scratch2); + return true; +} + +template +bool BaselineCodeGen::emit_Iter() { + frame.popRegsAndSync(1); + + if (!emitNextIC()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_MoreIter() { + frame.syncStack(0); + + masm.unboxObject(frame.addressOfStackValue(-1), R1.scratchReg()); + + masm.iteratorMore(R1.scratchReg(), R0, R2.scratchReg()); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emitIsMagicValue() { + frame.syncStack(0); + + Label isMagic, done; + masm.branchTestMagic(Assembler::Equal, frame.addressOfStackValue(-1), + &isMagic); + masm.moveValue(BooleanValue(false), R0); + masm.jump(&done); + + masm.bind(&isMagic); + masm.moveValue(BooleanValue(true), R0); + + masm.bind(&done); + frame.push(R0, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_IsNoIter() { + return emitIsMagicValue(); +} + +template +bool BaselineCodeGen::emit_EndIter() { + // Pop iterator value. + frame.pop(); + + // Pop the iterator object to close in R0. + frame.popRegsAndSync(1); + + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + MOZ_ASSERT(!regs.has(FramePointer)); + if (HasInterpreterPCReg()) { + regs.take(InterpreterPCReg); + } + + Register obj = R0.scratchReg(); + regs.take(obj); + masm.unboxObject(R0, obj); + + Register temp1 = regs.takeAny(); + Register temp2 = regs.takeAny(); + Register temp3 = regs.takeAny(); + masm.iteratorClose(obj, temp1, temp2, temp3); + return true; +} + +template +bool BaselineCodeGen::emit_CloseIter() { + frame.popRegsAndSync(1); + + Register iter = R0.scratchReg(); + masm.unboxObject(R0, iter); + + return emitNextIC(); +} + +template +bool BaselineCodeGen::emit_IsGenClosing() { + return emitIsMagicValue(); +} + +template +bool BaselineCodeGen::emit_IsNullOrUndefined() { + frame.syncStack(0); + + Label isNullOrUndefined, done; + masm.branchTestNull(Assembler::Equal, frame.addressOfStackValue(-1), + &isNullOrUndefined); + masm.branchTestUndefined(Assembler::Equal, frame.addressOfStackValue(-1), + &isNullOrUndefined); + masm.moveValue(BooleanValue(false), R0); + masm.jump(&done); + + masm.bind(&isNullOrUndefined); + masm.moveValue(BooleanValue(true), R0); + + masm.bind(&done); + frame.push(R0, JSVAL_TYPE_BOOLEAN); + return true; +} + +template +bool BaselineCodeGen::emit_GetRval() { + frame.syncStack(0); + + emitLoadReturnValue(R0); + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_SetRval() { + // Store to the frame's return value slot. + frame.storeStackValue(-1, frame.addressOfReturnValue(), R2); + masm.or32(Imm32(BaselineFrame::HAS_RVAL), frame.addressOfFlags()); + frame.pop(); + return true; +} + +template +bool BaselineCodeGen::emit_Callee() { + MOZ_ASSERT_IF(handler.maybeScript(), handler.maybeScript()->function()); + frame.syncStack(0); + masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(), + R0.scratchReg()); + masm.tagValue(JSVAL_TYPE_OBJECT, R0.scratchReg(), R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_EnvCallee() { + frame.syncStack(0); + uint8_t numHops = GET_UINT8(handler.pc()); + Register scratch = R0.scratchReg(); + + masm.loadPtr(frame.addressOfEnvironmentChain(), scratch); + for (unsigned i = 0; i < numHops; i++) { + Address nextAddr(scratch, + EnvironmentObject::offsetOfEnclosingEnvironment()); + masm.unboxObject(nextAddr, scratch); + } + + masm.loadValue(Address(scratch, CallObject::offsetOfCallee()), R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_EnvCallee() { + Register scratch = R0.scratchReg(); + Register env = R1.scratchReg(); + + static_assert(JSOpLength_EnvCallee - sizeof(jsbytecode) == ENVCOORD_HOPS_LEN, + "op must have uint8 operand for LoadAliasedVarEnv"); + + // Load the right environment object. + masm.loadPtr(frame.addressOfEnvironmentChain(), env); + LoadAliasedVarEnv(masm, env, scratch); + + masm.pushValue(Address(env, CallObject::offsetOfCallee())); + return true; +} + +template +bool BaselineCodeGen::emit_SuperBase() { + frame.popRegsAndSync(1); + + Register scratch = R0.scratchReg(); + Register proto = R1.scratchReg(); + + // Unbox callee. + masm.unboxObject(R0, scratch); + + // Load [[HomeObject]] + Address homeObjAddr(scratch, + FunctionExtended::offsetOfMethodHomeObjectSlot()); + + masm.assertFunctionIsExtended(scratch); +#ifdef DEBUG + Label isObject; + masm.branchTestObject(Assembler::Equal, homeObjAddr, &isObject); + masm.assumeUnreachable("[[HomeObject]] must be Object"); + masm.bind(&isObject); +#endif + masm.unboxObject(homeObjAddr, scratch); + + // Load prototype from [[HomeObject]] + masm.loadObjProto(scratch, proto); + +#ifdef DEBUG + // We won't encounter a lazy proto, because the prototype is guaranteed to + // either be a JSFunction or a PlainObject, and only proxy objects can have a + // lazy proto. + MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1); + + Label proxyCheckDone; + masm.branchPtr(Assembler::NotEqual, proto, ImmWord(1), &proxyCheckDone); + masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperBase"); + masm.bind(&proxyCheckDone); +#endif + + Label nullProto, done; + masm.branchPtr(Assembler::Equal, proto, ImmWord(0), &nullProto); + + // Box prototype and return + masm.tagValue(JSVAL_TYPE_OBJECT, proto, R1); + masm.jump(&done); + + masm.bind(&nullProto); + masm.moveValue(NullValue(), R1); + + masm.bind(&done); + frame.push(R1); + return true; +} + +template +bool BaselineCodeGen::emit_SuperFun() { + frame.popRegsAndSync(1); + + Register callee = R0.scratchReg(); + Register proto = R1.scratchReg(); +#ifdef DEBUG + Register scratch = R2.scratchReg(); +#endif + + // Unbox callee. + masm.unboxObject(R0, callee); + +#ifdef DEBUG + Label classCheckDone; + masm.branchTestObjIsFunction(Assembler::Equal, callee, scratch, callee, + &classCheckDone); + masm.assumeUnreachable("Unexpected non-JSFunction callee in JSOp::SuperFun"); + masm.bind(&classCheckDone); +#endif + + // Load prototype of callee + masm.loadObjProto(callee, proto); + +#ifdef DEBUG + // We won't encounter a lazy proto, because |callee| is guaranteed to be a + // JSFunction and only proxy objects can have a lazy proto. + MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1); + + Label proxyCheckDone; + masm.branchPtr(Assembler::NotEqual, proto, ImmWord(1), &proxyCheckDone); + masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperFun"); + masm.bind(&proxyCheckDone); +#endif + + Label nullProto, done; + masm.branchPtr(Assembler::Equal, proto, ImmWord(0), &nullProto); + + // Box prototype and return + masm.tagValue(JSVAL_TYPE_OBJECT, proto, R1); + masm.jump(&done); + + masm.bind(&nullProto); + masm.moveValue(NullValue(), R1); + + masm.bind(&done); + frame.push(R1); + return true; +} + +template +bool BaselineCodeGen::emit_Arguments() { + frame.syncStack(0); + + MOZ_ASSERT_IF(handler.maybeScript(), handler.maybeScript()->needsArgsObj()); + + prepareVMCall(); + + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, BaselineFrame*, MutableHandleValue); + if (!callVM()) { + return false; + } + + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_Rest() { + frame.syncStack(0); + + if (!emitNextIC()) { + return false; + } + + // Mark R0 as pushed stack value. + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_Generator() { + frame.assertStackDepth(0); + + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + + prepareVMCall(); + pushArg(R0.scratchReg()); + + using Fn = JSObject* (*)(JSContext*, BaselineFrame*); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emitSuspend(JSOp op) { + MOZ_ASSERT(op == JSOp::InitialYield || op == JSOp::Yield || + op == JSOp::Await); + + // Load the generator object in R2, but leave the return value on the + // expression stack. + Register genObj = R2.scratchReg(); + if (op == JSOp::InitialYield) { + // Generator and return value are one and the same. + frame.syncStack(0); + frame.assertStackDepth(1); + masm.unboxObject(frame.addressOfStackValue(-1), genObj); + } else { + frame.popRegsAndSync(1); + masm.unboxObject(R0, genObj); + } + + if (frame.hasKnownStackDepth(1) && !handler.canHaveFixedSlots()) { + // If the expression stack is empty, we can inline the Yield. Note that this + // branch is never taken for the interpreter because it doesn't know static + // stack depths. + MOZ_ASSERT_IF(op == JSOp::InitialYield && handler.maybePC(), + GET_RESUMEINDEX(handler.maybePC()) == 0); + Address resumeIndexSlot(genObj, + AbstractGeneratorObject::offsetOfResumeIndexSlot()); + Register temp = R1.scratchReg(); + if (op == JSOp::InitialYield) { + masm.storeValue(Int32Value(0), resumeIndexSlot); + } else { + jsbytecode* pc = handler.maybePC(); + MOZ_ASSERT(pc, "compiler-only code never has a null pc"); + masm.move32(Imm32(GET_RESUMEINDEX(pc)), temp); + masm.storeValue(JSVAL_TYPE_INT32, temp, resumeIndexSlot); + } + + Register envObj = R0.scratchReg(); + Address envChainSlot( + genObj, AbstractGeneratorObject::offsetOfEnvironmentChainSlot()); + masm.loadPtr(frame.addressOfEnvironmentChain(), envObj); + masm.guardedCallPreBarrierAnyZone(envChainSlot, MIRType::Value, temp); + masm.storeValue(JSVAL_TYPE_OBJECT, envObj, envChainSlot); + + Label skipBarrier; + masm.branchPtrInNurseryChunk(Assembler::Equal, genObj, temp, &skipBarrier); + masm.branchPtrInNurseryChunk(Assembler::NotEqual, envObj, temp, + &skipBarrier); + MOZ_ASSERT(genObj == R2.scratchReg()); + masm.call(&postBarrierSlot_); + masm.bind(&skipBarrier); + } else { + masm.loadBaselineFramePtr(FramePointer, R1.scratchReg()); + computeFrameSize(R0.scratchReg()); + + prepareVMCall(); + pushBytecodePCArg(); + pushArg(R0.scratchReg()); + pushArg(R1.scratchReg()); + pushArg(genObj); + + using Fn = bool (*)(JSContext*, HandleObject, BaselineFrame*, uint32_t, + const jsbytecode*); + if (!callVM()) { + return false; + } + } + + masm.loadValue(frame.addressOfStackValue(-1), JSReturnOperand); + if (!emitReturn()) { + return false; + } + + // Three values are pushed onto the stack when resuming the generator, + // replacing the one slot that holds the return value. + frame.incStackDepth(2); + return true; +} + +template +bool BaselineCodeGen::emit_InitialYield() { + return emitSuspend(JSOp::InitialYield); +} + +template +bool BaselineCodeGen::emit_Yield() { + return emitSuspend(JSOp::Yield); +} + +template +bool BaselineCodeGen::emit_Await() { + return emitSuspend(JSOp::Await); +} + +template <> +template +bool BaselineCompilerCodeGen::emitAfterYieldDebugInstrumentation( + const F& ifDebuggee, Register) { + if (handler.compileDebugInstrumentation()) { + return ifDebuggee(); + } + return true; +} + +template <> +template +bool BaselineInterpreterCodeGen::emitAfterYieldDebugInstrumentation( + const F& ifDebuggee, Register scratch) { + // Note that we can't use emitDebugInstrumentation here because the frame's + // DEBUGGEE flag hasn't been initialized yet. + + // If the current Realm is not a debuggee we're done. + Label done; + CodeOffset toggleOffset = masm.toggledJump(&done); + if (!handler.addDebugInstrumentationOffset(cx, toggleOffset)) { + return false; + } + masm.loadPtr(AbsoluteAddress(cx->addressOfRealm()), scratch); + masm.branchTest32(Assembler::Zero, + Address(scratch, Realm::offsetOfDebugModeBits()), + Imm32(Realm::debugModeIsDebuggeeBit()), &done); + + if (!ifDebuggee()) { + return false; + } + + masm.bind(&done); + return true; +} + +template +bool BaselineCodeGen::emit_AfterYield() { + if (!emit_JumpTarget()) { + return false; + } + + auto ifDebuggee = [this]() { + frame.assertSyncedStack(); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + prepareVMCall(); + pushArg(R0.scratchReg()); + + const RetAddrEntry::Kind kind = RetAddrEntry::Kind::DebugAfterYield; + + using Fn = bool (*)(JSContext*, BaselineFrame*); + if (!callVM(kind)) { + return false; + } + + return true; + }; + return emitAfterYieldDebugInstrumentation(ifDebuggee, R0.scratchReg()); +} + +template +bool BaselineCodeGen::emit_FinalYieldRval() { + // Store generator in R0. + frame.popRegsAndSync(1); + masm.unboxObject(R0, R0.scratchReg()); + + prepareVMCall(); + pushBytecodePCArg(); + pushArg(R0.scratchReg()); + + using Fn = bool (*)(JSContext*, HandleObject, const jsbytecode*); + if (!callVM()) { + return false; + } + + masm.loadValue(frame.addressOfReturnValue(), JSReturnOperand); + return emitReturn(); +} + +template <> +void BaselineCompilerCodeGen::emitJumpToInterpretOpLabel() { + TrampolinePtr code = + cx->runtime()->jitRuntime()->baselineInterpreter().interpretOpAddr(); + masm.jump(code); +} + +template <> +void BaselineInterpreterCodeGen::emitJumpToInterpretOpLabel() { + masm.jump(handler.interpretOpLabel()); +} + +template +bool BaselineCodeGen::emitEnterGeneratorCode(Register script, + Register resumeIndex, + Register scratch) { + // Resume in either the BaselineScript (if present) or Baseline Interpreter. + + static_assert(BaselineDisabledScript == 0x1, + "Comparison below requires specific sentinel encoding"); + + // Initialize the icScript slot in the baseline frame. + masm.loadJitScript(script, scratch); + masm.computeEffectiveAddress(Address(scratch, JitScript::offsetOfICScript()), + scratch); + Address icScriptAddr(FramePointer, BaselineFrame::reverseOffsetOfICScript()); + masm.storePtr(scratch, icScriptAddr); + + Label noBaselineScript; + masm.loadJitScript(script, scratch); + masm.loadPtr(Address(scratch, JitScript::offsetOfBaselineScript()), scratch); + masm.branchPtr(Assembler::BelowOrEqual, scratch, + ImmPtr(BaselineDisabledScriptPtr), &noBaselineScript); + + masm.load32(Address(scratch, BaselineScript::offsetOfResumeEntriesOffset()), + script); + masm.addPtr(scratch, script); + masm.loadPtr( + BaseIndex(script, resumeIndex, ScaleFromElemWidth(sizeof(uintptr_t))), + scratch); + masm.jump(scratch); + + masm.bind(&noBaselineScript); + + // Initialize interpreter frame fields. + Address flagsAddr(FramePointer, BaselineFrame::reverseOffsetOfFlags()); + Address scriptAddr(FramePointer, + BaselineFrame::reverseOffsetOfInterpreterScript()); + masm.or32(Imm32(BaselineFrame::RUNNING_IN_INTERPRETER), flagsAddr); + masm.storePtr(script, scriptAddr); + + // Initialize pc and jump to it. + emitInterpJumpToResumeEntry(script, resumeIndex, scratch); + return true; +} + +template +bool BaselineCodeGen::emit_Resume() { + frame.syncStack(0); + masm.assertStackAlignment(sizeof(Value), 0); + + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + MOZ_ASSERT(!regs.has(FramePointer)); + if (HasInterpreterPCReg()) { + regs.take(InterpreterPCReg); + } + + saveInterpreterPCReg(); + + // Load generator object. + Register genObj = regs.takeAny(); + masm.unboxObject(frame.addressOfStackValue(-3), genObj); + + // Load callee. + Register callee = regs.takeAny(); + masm.unboxObject( + Address(genObj, AbstractGeneratorObject::offsetOfCalleeSlot()), callee); + + // Save a pointer to the JSOp::Resume operand stack Values. + Register callerStackPtr = regs.takeAny(); + masm.computeEffectiveAddress(frame.addressOfStackValue(-1), callerStackPtr); + + // Branch to |interpret| to resume the generator in the C++ interpreter if the + // script does not have a JitScript. + Label interpret; + Register scratch1 = regs.takeAny(); + masm.loadPrivate(Address(callee, JSFunction::offsetOfJitInfoOrScript()), + scratch1); + masm.branchIfScriptHasNoJitScript(scratch1, &interpret); + + // Push |undefined| for all formals. + Register scratch2 = regs.takeAny(); + Label loop, loopDone; + masm.loadFunctionArgCount(callee, scratch2); + + static_assert(sizeof(Value) == 8); + static_assert(JitStackAlignment == 16 || JitStackAlignment == 8); + // If JitStackValueAlignment == 1, then we were already correctly aligned on + // entry, as guaranteed by the assertStackAlignment at the entry to this + // function. + if (JitStackValueAlignment > 1) { + Register alignment = regs.takeAny(); + masm.moveStackPtrTo(alignment); + masm.alignJitStackBasedOnNArgs(scratch2, false); + + // Compute alignment adjustment. + masm.subStackPtrFrom(alignment); + + // Some code, like BaselineFrame::trace, will inspect the whole range of + // the stack frame. In order to ensure that garbage data left behind from + // previous activations doesn't confuse other machinery, we zero out the + // alignment bytes. + Label alignmentZero; + masm.branchPtr(Assembler::Equal, alignment, ImmWord(0), &alignmentZero); + + // Since we know prior to the stack alignment that the stack was 8 byte + // aligned, and JitStackAlignment is 8 or 16 bytes, if we are doing an + // alignment then we -must- have aligned by subtracting 8 bytes from + // the stack pointer. + // + // So we can freely store a valid double here. + masm.storeValue(DoubleValue(0), Address(masm.getStackPointer(), 0)); + masm.bind(&alignmentZero); + } + + masm.branchTest32(Assembler::Zero, scratch2, scratch2, &loopDone); + masm.bind(&loop); + { + masm.pushValue(UndefinedValue()); + masm.branchSub32(Assembler::NonZero, Imm32(1), scratch2, &loop); + } + masm.bind(&loopDone); + + // Push |undefined| for |this|. + masm.pushValue(UndefinedValue()); + +#ifdef DEBUG + // Update BaselineFrame debugFrameSize field. + masm.mov(FramePointer, scratch2); + masm.subStackPtrFrom(scratch2); + masm.store32(scratch2, frame.addressOfDebugFrameSize()); +#endif + + masm.PushCalleeToken(callee, /* constructing = */ false); + masm.pushFrameDescriptorForJitCall(FrameType::BaselineJS, /* argc = */ 0); + + // PushCalleeToken bumped framePushed. Reset it. + MOZ_ASSERT(masm.framePushed() == sizeof(uintptr_t)); + masm.setFramePushed(0); + + regs.add(callee); + + // Push a fake return address on the stack. We will resume here when the + // generator returns. + Label genStart, returnTarget; +#ifdef JS_USE_LINK_REGISTER + masm.call(&genStart); +#else + masm.callAndPushReturnAddress(&genStart); +#endif + + // Record the return address so the return offset -> pc mapping works. + if (!handler.recordCallRetAddr(cx, RetAddrEntry::Kind::IC, + masm.currentOffset())) { + return false; + } + + masm.jump(&returnTarget); + masm.bind(&genStart); +#ifdef JS_USE_LINK_REGISTER + masm.pushReturnAddress(); +#endif + + // Construct BaselineFrame. + masm.push(FramePointer); + masm.moveStackPtrTo(FramePointer); + + // If profiler instrumentation is on, update lastProfilingFrame on + // current JitActivation + { + Register scratchReg = scratch2; + Label skip; + AbsoluteAddress addressOfEnabled( + cx->runtime()->geckoProfiler().addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skip); + masm.loadJSContext(scratchReg); + masm.loadPtr(Address(scratchReg, JSContext::offsetOfProfilingActivation()), + scratchReg); + masm.storePtr( + FramePointer, + Address(scratchReg, JitActivation::offsetOfLastProfilingFrame())); + masm.bind(&skip); + } + + masm.subFromStackPtr(Imm32(BaselineFrame::Size())); + masm.assertStackAlignment(sizeof(Value), 0); + + // Store flags and env chain. + masm.store32(Imm32(BaselineFrame::HAS_INITIAL_ENV), frame.addressOfFlags()); + masm.unboxObject( + Address(genObj, AbstractGeneratorObject::offsetOfEnvironmentChainSlot()), + scratch2); + masm.storePtr(scratch2, frame.addressOfEnvironmentChain()); + + // Store the arguments object if there is one. + Label noArgsObj; + Address argsObjSlot(genObj, AbstractGeneratorObject::offsetOfArgsObjSlot()); + masm.fallibleUnboxObject(argsObjSlot, scratch2, &noArgsObj); + { + masm.storePtr(scratch2, frame.addressOfArgsObj()); + masm.or32(Imm32(BaselineFrame::HAS_ARGS_OBJ), frame.addressOfFlags()); + } + masm.bind(&noArgsObj); + + // Push locals and expression slots if needed. + Label noStackStorage; + Address stackStorageSlot(genObj, + AbstractGeneratorObject::offsetOfStackStorageSlot()); + masm.fallibleUnboxObject(stackStorageSlot, scratch2, &noStackStorage); + { + Register initLength = regs.takeAny(); + masm.loadPtr(Address(scratch2, NativeObject::offsetOfElements()), scratch2); + masm.load32(Address(scratch2, ObjectElements::offsetOfInitializedLength()), + initLength); + masm.store32( + Imm32(0), + Address(scratch2, ObjectElements::offsetOfInitializedLength())); + + Label loop, loopDone; + masm.branchTest32(Assembler::Zero, initLength, initLength, &loopDone); + masm.bind(&loop); + { + masm.pushValue(Address(scratch2, 0)); + masm.guardedCallPreBarrierAnyZone(Address(scratch2, 0), MIRType::Value, + scratch1); + masm.addPtr(Imm32(sizeof(Value)), scratch2); + masm.branchSub32(Assembler::NonZero, Imm32(1), initLength, &loop); + } + masm.bind(&loopDone); + regs.add(initLength); + } + + masm.bind(&noStackStorage); + + // Push arg, generator, resumeKind stack Values, in that order. + masm.pushValue(Address(callerStackPtr, sizeof(Value))); + masm.pushValue(JSVAL_TYPE_OBJECT, genObj); + masm.pushValue(Address(callerStackPtr, 0)); + + masm.switchToObjectRealm(genObj, scratch2); + + // Load script in scratch1. + masm.unboxObject( + Address(genObj, AbstractGeneratorObject::offsetOfCalleeSlot()), scratch1); + masm.loadPrivate(Address(scratch1, JSFunction::offsetOfJitInfoOrScript()), + scratch1); + + // Load resume index in scratch2 and mark generator as running. + Address resumeIndexSlot(genObj, + AbstractGeneratorObject::offsetOfResumeIndexSlot()); + masm.unboxInt32(resumeIndexSlot, scratch2); + masm.storeValue(Int32Value(AbstractGeneratorObject::RESUME_INDEX_RUNNING), + resumeIndexSlot); + + if (!emitEnterGeneratorCode(scratch1, scratch2, regs.getAny())) { + return false; + } + + // Call into the VM to resume the generator in the C++ interpreter if there's + // no JitScript. + masm.bind(&interpret); + + prepareVMCall(); + + pushArg(callerStackPtr); + pushArg(genObj); + + using Fn = bool (*)(JSContext*, HandleObject, Value*, MutableHandleValue); + if (!callVM()) { + return false; + } + + masm.bind(&returnTarget); + + // Restore Stack pointer + masm.computeEffectiveAddress(frame.addressOfStackValue(-1), + masm.getStackPointer()); + + // After the generator returns, we restore the stack pointer, switch back to + // the current realm, push the return value, and we're done. + if (JSScript* script = handler.maybeScript()) { + masm.switchToRealm(script->realm(), R2.scratchReg()); + } else { + masm.switchToBaselineFrameRealm(R2.scratchReg()); + } + restoreInterpreterPCReg(); + frame.popn(3); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_CheckResumeKind() { + // Load resumeKind in R1, generator in R0. + frame.popRegsAndSync(2); + +#ifdef DEBUG + Label ok; + masm.branchTestInt32(Assembler::Equal, R1, &ok); + masm.assumeUnreachable("Expected int32 resumeKind"); + masm.bind(&ok); +#endif + + // If resumeKind is 'next' we don't have to do anything. + Label done; + masm.unboxInt32(R1, R1.scratchReg()); + masm.branch32(Assembler::Equal, R1.scratchReg(), + Imm32(int32_t(GeneratorResumeKind::Next)), &done); + + prepareVMCall(); + + pushArg(R1.scratchReg()); // resumeKind + + masm.loadValue(frame.addressOfStackValue(-1), R2); + pushArg(R2); // arg + + masm.unboxObject(R0, R0.scratchReg()); + pushArg(R0.scratchReg()); // genObj + + masm.loadBaselineFramePtr(FramePointer, R2.scratchReg()); + pushArg(R2.scratchReg()); // frame + + using Fn = bool (*)(JSContext*, BaselineFrame*, + Handle, HandleValue, int32_t); + if (!callVM()) { + return false; + } + + masm.bind(&done); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_ResumeKind() { + GeneratorResumeKind resumeKind = ResumeKindFromPC(handler.pc()); + frame.push(Int32Value(int32_t(resumeKind))); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_ResumeKind() { + LoadUint8Operand(masm, R0.scratchReg()); + masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_DebugCheckSelfHosted() { +#ifdef DEBUG + frame.syncStack(0); + + masm.loadValue(frame.addressOfStackValue(-1), R0); + + prepareVMCall(); + pushArg(R0); + + using Fn = bool (*)(JSContext*, HandleValue); + if (!callVM()) { + return false; + } +#endif + return true; +} + +template +bool BaselineCodeGen::emit_IsConstructing() { + frame.push(MagicValue(JS_IS_CONSTRUCTING)); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_JumpTarget() { + MaybeIncrementCodeCoverageCounter(masm, handler.script(), handler.pc()); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_JumpTarget() { + Register scratch1 = R0.scratchReg(); + Register scratch2 = R1.scratchReg(); + + Label skipCoverage; + CodeOffset toggleOffset = masm.toggledJump(&skipCoverage); + masm.call(handler.codeCoverageAtPCLabel()); + masm.bind(&skipCoverage); + if (!handler.codeCoverageOffsets().append(toggleOffset.offset())) { + return false; + } + + // Load icIndex in scratch1. + LoadInt32Operand(masm, scratch1); + + // Compute ICEntry* and store to frame->interpreterICEntry. + masm.loadPtr(frame.addressOfICScript(), scratch2); + static_assert(sizeof(ICEntry) == sizeof(uintptr_t)); + masm.computeEffectiveAddress(BaseIndex(scratch2, scratch1, ScalePointer, + ICScript::offsetOfICEntries()), + scratch2); + masm.storePtr(scratch2, frame.addressOfInterpreterICEntry()); + return true; +} + +template +bool BaselineCodeGen::emit_CheckClassHeritage() { + frame.syncStack(0); + + // Leave the heritage value on the stack. + masm.loadValue(frame.addressOfStackValue(-1), R0); + + prepareVMCall(); + pushArg(R0); + + using Fn = bool (*)(JSContext*, HandleValue); + return callVM(); +} + +template +bool BaselineCodeGen::emit_InitHomeObject() { + // Load HomeObject in R0. + frame.popRegsAndSync(1); + + // Load function off stack + Register func = R2.scratchReg(); + masm.unboxObject(frame.addressOfStackValue(-1), func); + + masm.assertFunctionIsExtended(func); + + // Set HOMEOBJECT_SLOT + Register temp = R1.scratchReg(); + Address addr(func, FunctionExtended::offsetOfMethodHomeObjectSlot()); + masm.guardedCallPreBarrierAnyZone(addr, MIRType::Value, temp); + masm.storeValue(R0, addr); + + Label skipBarrier; + masm.branchPtrInNurseryChunk(Assembler::Equal, func, temp, &skipBarrier); + masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier); + masm.call(&postBarrierSlot_); + masm.bind(&skipBarrier); + + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_BuiltinObject() { + // Built-in objects are constants for a given global. + auto kind = BuiltinObjectKind(GET_UINT8(handler.pc())); + JSObject* builtin = BuiltinObjectOperation(cx, kind); + if (!builtin) { + return false; + } + frame.push(ObjectValue(*builtin)); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_BuiltinObject() { + prepareVMCall(); + + pushUint8BytecodeOperandArg(R0.scratchReg()); + + using Fn = JSObject* (*)(JSContext*, BuiltinObjectKind); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_ObjWithProto() { + frame.syncStack(0); + + // Leave the proto value on the stack for the decompiler + masm.loadValue(frame.addressOfStackValue(-1), R0); + + prepareVMCall(); + pushArg(R0); + + using Fn = PlainObject* (*)(JSContext*, HandleValue); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.pop(); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_FunWithProto() { + frame.popRegsAndSync(1); + + masm.unboxObject(R0, R0.scratchReg()); + masm.loadPtr(frame.addressOfEnvironmentChain(), R1.scratchReg()); + + prepareVMCall(); + pushArg(R0.scratchReg()); + pushArg(R1.scratchReg()); + pushScriptGCThingArg(ScriptGCThingType::Function, R0.scratchReg(), + R1.scratchReg()); + + using Fn = + JSObject* (*)(JSContext*, HandleFunction, HandleObject, HandleObject); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_ImportMeta() { + // Note: this is like the interpreter implementation, but optimized a bit by + // calling GetModuleObjectForScript at compile-time. + + Rooted module(cx, GetModuleObjectForScript(handler.script())); + MOZ_ASSERT(module); + + frame.syncStack(0); + + prepareVMCall(); + pushArg(ImmGCPtr(module)); + + using Fn = JSObject* (*)(JSContext*, HandleObject); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineInterpreterCodeGen::emit_ImportMeta() { + prepareVMCall(); + + pushScriptArg(); + + using Fn = JSObject* (*)(JSContext*, HandleScript); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.push(R0); + return true; +} + +template +bool BaselineCodeGen::emit_DynamicImport() { + // Put specifier into R0 and object value into R1 + frame.popRegsAndSync(2); + + prepareVMCall(); + pushArg(R1); + pushArg(R0); + pushScriptArg(); + + using Fn = JSObject* (*)(JSContext*, HandleScript, HandleValue, HandleValue); + if (!callVM()) { + return false; + } + + masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0); + frame.push(R0); + return true; +} + +template <> +bool BaselineCompilerCodeGen::emit_ForceInterpreter() { + // Caller is responsible for checking script->hasForceInterpreterOp(). + MOZ_CRASH("JSOp::ForceInterpreter in baseline"); +} + +template <> +bool BaselineInterpreterCodeGen::emit_ForceInterpreter() { + masm.assumeUnreachable("JSOp::ForceInterpreter"); + return true; +} + +template +bool BaselineCodeGen::emitPrologue() { + AutoCreatedBy acb(masm, "BaselineCodeGen::emitPrologue"); + +#ifdef JS_USE_LINK_REGISTER + // Push link register from generateEnterJIT()'s BLR. + masm.pushReturnAddress(); +#endif + + masm.push(FramePointer); + masm.moveStackPtrTo(FramePointer); + + masm.checkStackAlignment(); + + emitProfilerEnterFrame(); + + masm.subFromStackPtr(Imm32(BaselineFrame::Size())); + + // Initialize BaselineFrame. Also handles env chain pre-initialization (in + // case GC gets run during stack check). For global and eval scripts, the env + // chain is in R1. For function scripts, the env chain is in the callee. + emitInitFrameFields(R1.scratchReg()); + + // When compiling with Debugger instrumentation, set the debuggeeness of + // the frame before any operation that can call into the VM. + if (!emitIsDebuggeeCheck()) { + return false; + } + + // Initialize the env chain before any operation that may call into the VM and + // trigger a GC. + if (!initEnvironmentChain()) { + return false; + } + + // Check for overrecursion before initializing locals. + if (!emitStackCheck()) { + return false; + } + + emitInitializeLocals(); + + // Ion prologue bailouts will enter here in the Baseline Interpreter. + masm.bind(&bailoutPrologue_); + + frame.assertSyncedStack(); + + if (JSScript* script = handler.maybeScript()) { + masm.debugAssertContextRealm(script->realm(), R1.scratchReg()); + } + + if (!emitDebugPrologue()) { + return false; + } + + if (!emitHandleCodeCoverageAtPrologue()) { + return false; + } + + if (!emitWarmUpCounterIncrement()) { + return false; + } + + warmUpCheckPrologueOffset_ = CodeOffset(masm.currentOffset()); + + return true; +} + +template +bool BaselineCodeGen::emitEpilogue() { + AutoCreatedBy acb(masm, "BaselineCodeGen::emitEpilogue"); + + masm.bind(&return_); + + if (!handler.shouldEmitDebugEpilogueAtReturnOp()) { + if (!emitDebugEpilogue()) { + return false; + } + } + + emitProfilerExitFrame(); + + masm.moveToStackPtr(FramePointer); + masm.pop(FramePointer); + + masm.ret(); + return true; +} + +MethodStatus BaselineCompiler::emitBody() { + AutoCreatedBy acb(masm, "BaselineCompiler::emitBody"); + + JSScript* script = handler.script(); + MOZ_ASSERT(handler.pc() == script->code()); + + mozilla::DebugOnly prevpc = handler.pc(); + + while (true) { + JSOp op = JSOp(*handler.pc()); + JitSpew(JitSpew_BaselineOp, "Compiling op @ %d: %s", + int(script->pcToOffset(handler.pc())), CodeName(op)); + + BytecodeInfo* info = handler.analysis().maybeInfo(handler.pc()); + + // Skip unreachable ops. + if (!info) { + // Test if last instructions and stop emitting in that case. + handler.moveToNextPC(); + if (handler.pc() >= script->codeEnd()) { + break; + } + + prevpc = handler.pc(); + continue; + } + + if (info->jumpTarget) { + // Fully sync the stack if there are incoming jumps. + frame.syncStack(0); + frame.setStackDepth(info->stackDepth); + masm.bind(handler.labelOf(handler.pc())); + } else if (MOZ_UNLIKELY(compileDebugInstrumentation())) { + // Also fully sync the stack if the debugger is enabled. + frame.syncStack(0); + } else { + // At the beginning of any op, at most the top 2 stack-values are + // unsynced. + if (frame.stackDepth() > 2) { + frame.syncStack(2); + } + } + + frame.assertValidState(*info); + + // If the script has a resume offset for this pc we need to keep track of + // the native code offset. + if (info->hasResumeOffset) { + frame.assertSyncedStack(); + uint32_t pcOffset = script->pcToOffset(handler.pc()); + uint32_t nativeOffset = masm.currentOffset(); + if (!resumeOffsetEntries_.emplaceBack(pcOffset, nativeOffset)) { + ReportOutOfMemory(cx); + return Method_Error; + } + } + + // Emit traps for breakpoints and step mode. + if (MOZ_UNLIKELY(compileDebugInstrumentation()) && !emitDebugTrap()) { + return Method_Error; + } + + perfSpewer_.recordInstruction(cx, masm, handler.pc(), frame); + +#define EMIT_OP(OP, ...) \ + case JSOp::OP: { \ + AutoCreatedBy acb(masm, "op=" #OP); \ + if (MOZ_UNLIKELY(!this->emit_##OP())) return Method_Error; \ + } break; + + switch (op) { + FOR_EACH_OPCODE(EMIT_OP) + default: + MOZ_CRASH("Unexpected op"); + } + +#undef EMIT_OP + + MOZ_ASSERT(masm.framePushed() == 0); + + // Test if last instructions and stop emitting in that case. + handler.moveToNextPC(); + if (handler.pc() >= script->codeEnd()) { + break; + } + +#ifdef DEBUG + prevpc = handler.pc(); +#endif + } + + MOZ_ASSERT(JSOp(*prevpc) == JSOp::RetRval || JSOp(*prevpc) == JSOp::Return); + return Method_Compiled; +} + +bool BaselineInterpreterGenerator::emitDebugTrap() { + CodeOffset offset = masm.nopPatchableToCall(); + if (!debugTrapOffsets_.append(offset.offset())) { + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +// Register holding the bytecode pc during dispatch. This exists so the debug +// trap handler can reload the pc into this register when it's done. +static constexpr Register InterpreterPCRegAtDispatch = + HasInterpreterPCReg() ? InterpreterPCReg : R0.scratchReg(); + +bool BaselineInterpreterGenerator::emitInterpreterLoop() { + AutoCreatedBy acb(masm, "BaselineInterpreterGenerator::emitInterpreterLoop"); + + Register scratch1 = R0.scratchReg(); + Register scratch2 = R1.scratchReg(); + + // Entry point for interpreting a bytecode op. No registers are live except + // for InterpreterPCReg. + masm.bind(handler.interpretOpWithPCRegLabel()); + + // Emit a patchable call for debugger breakpoints/stepping. + if (!emitDebugTrap()) { + return false; + } + Label interpretOpAfterDebugTrap; + masm.bind(&interpretOpAfterDebugTrap); + + // Load pc, bytecode op. + Register pcReg = LoadBytecodePC(masm, scratch1); + masm.load8ZeroExtend(Address(pcReg, 0), scratch1); + + // Jump to table[op]. + { + CodeOffset label = masm.moveNearAddressWithPatch(scratch2); + if (!tableLabels_.append(label)) { + return false; + } + BaseIndex pointer(scratch2, scratch1, ScalePointer); + masm.branchToComputedAddress(pointer); + } + + // At the end of each op, emit code to bump the pc and jump to the + // next op (this is also known as a threaded interpreter). + auto opEpilogue = [&](JSOp op, size_t opLength) -> bool { + MOZ_ASSERT(masm.framePushed() == 0); + + if (!BytecodeFallsThrough(op)) { + // Nothing to do. + masm.assumeUnreachable("unexpected fall through"); + return true; + } + + // Bump frame->interpreterICEntry if needed. + if (BytecodeOpHasIC(op)) { + frame.bumpInterpreterICEntry(); + } + + // Bump bytecode PC. + if (HasInterpreterPCReg()) { + MOZ_ASSERT(InterpreterPCRegAtDispatch == InterpreterPCReg); + masm.addPtr(Imm32(opLength), InterpreterPCReg); + } else { + MOZ_ASSERT(InterpreterPCRegAtDispatch == scratch1); + masm.loadPtr(frame.addressOfInterpreterPC(), InterpreterPCRegAtDispatch); + masm.addPtr(Imm32(opLength), InterpreterPCRegAtDispatch); + masm.storePtr(InterpreterPCRegAtDispatch, frame.addressOfInterpreterPC()); + } + + if (!emitDebugTrap()) { + return false; + } + + // Load the opcode, jump to table[op]. + masm.load8ZeroExtend(Address(InterpreterPCRegAtDispatch, 0), scratch1); + CodeOffset label = masm.moveNearAddressWithPatch(scratch2); + if (!tableLabels_.append(label)) { + return false; + } + BaseIndex pointer(scratch2, scratch1, ScalePointer); + masm.branchToComputedAddress(pointer); + return true; + }; + + // Emit code for each bytecode op. + Label opLabels[JSOP_LIMIT]; +#define EMIT_OP(OP, ...) \ + { \ + AutoCreatedBy acb(masm, "op=" #OP); \ + perfSpewer_.recordOffset(masm, JSOp::OP); \ + masm.bind(&opLabels[uint8_t(JSOp::OP)]); \ + handler.setCurrentOp(JSOp::OP); \ + if (!this->emit_##OP()) { \ + return false; \ + } \ + if (!opEpilogue(JSOp::OP, JSOpLength_##OP)) { \ + return false; \ + } \ + handler.resetCurrentOp(); \ + } + FOR_EACH_OPCODE(EMIT_OP) +#undef EMIT_OP + + // External entry point to start interpreting bytecode ops. This is used for + // things like exception handling and OSR. DebugModeOSR patches JIT frames to + // return here from the DebugTrapHandler. + masm.bind(handler.interpretOpLabel()); + interpretOpOffset_ = masm.currentOffset(); + restoreInterpreterPCReg(); + masm.jump(handler.interpretOpWithPCRegLabel()); + + // Second external entry point: this skips the debug trap for the first op + // and is used by OSR. + interpretOpNoDebugTrapOffset_ = masm.currentOffset(); + restoreInterpreterPCReg(); + masm.jump(&interpretOpAfterDebugTrap); + + // External entry point for Ion prologue bailouts. + bailoutPrologueOffset_ = CodeOffset(masm.currentOffset()); + restoreInterpreterPCReg(); + masm.jump(&bailoutPrologue_); + + // Emit debug trap handler code (target of patchable call instructions). This + // is just a tail call to the debug trap handler trampoline code. + { + JitRuntime* jrt = cx->runtime()->jitRuntime(); + JitCode* handlerCode = + jrt->debugTrapHandler(cx, DebugTrapHandlerKind::Interpreter); + if (!handlerCode) { + return false; + } + + debugTrapHandlerOffset_ = masm.currentOffset(); + masm.jump(handlerCode); + } + + // Emit the table. + masm.haltingAlign(sizeof(void*)); + +#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) + size_t numInstructions = JSOP_LIMIT * (sizeof(uintptr_t) / sizeof(uint32_t)); + AutoForbidPoolsAndNops afp(&masm, numInstructions); +#endif + + tableOffset_ = masm.currentOffset(); + + for (size_t i = 0; i < JSOP_LIMIT; i++) { + const Label& opLabel = opLabels[i]; + MOZ_ASSERT(opLabel.bound()); + CodeLabel cl; + masm.writeCodePointer(&cl); + cl.target()->bind(opLabel.offset()); + masm.addCodeLabel(cl); + } + + return true; +} + +void BaselineInterpreterGenerator::emitOutOfLineCodeCoverageInstrumentation() { + AutoCreatedBy acb(masm, + "BaselineInterpreterGenerator::" + "emitOutOfLineCodeCoverageInstrumentation"); + + masm.bind(handler.codeCoverageAtPrologueLabel()); +#ifdef JS_USE_LINK_REGISTER + masm.pushReturnAddress(); +#endif + + saveInterpreterPCReg(); + + using Fn1 = void (*)(BaselineFrame* frame); + masm.setupUnalignedABICall(R0.scratchReg()); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + masm.passABIArg(R0.scratchReg()); + masm.callWithABI(); + + restoreInterpreterPCReg(); + masm.ret(); + + masm.bind(handler.codeCoverageAtPCLabel()); +#ifdef JS_USE_LINK_REGISTER + masm.pushReturnAddress(); +#endif + + saveInterpreterPCReg(); + + using Fn2 = void (*)(BaselineFrame* frame, jsbytecode* pc); + masm.setupUnalignedABICall(R0.scratchReg()); + masm.loadBaselineFramePtr(FramePointer, R0.scratchReg()); + masm.passABIArg(R0.scratchReg()); + Register pcReg = LoadBytecodePC(masm, R2.scratchReg()); + masm.passABIArg(pcReg); + masm.callWithABI(); + + restoreInterpreterPCReg(); + masm.ret(); +} + +bool BaselineInterpreterGenerator::generate(BaselineInterpreter& interpreter) { + AutoCreatedBy acb(masm, "BaselineInterpreterGenerator::generate"); + + perfSpewer_.recordOffset(masm, "Prologue"); + if (!emitPrologue()) { + return false; + } + + perfSpewer_.recordOffset(masm, "InterpreterLoop"); + if (!emitInterpreterLoop()) { + return false; + } + + perfSpewer_.recordOffset(masm, "Epilogue"); + if (!emitEpilogue()) { + return false; + } + + perfSpewer_.recordOffset(masm, "OOLPostBarrierSlot"); + if (!emitOutOfLinePostBarrierSlot()) { + return false; + } + + perfSpewer_.recordOffset(masm, "OOLCodeCoverageInstrumentation"); + emitOutOfLineCodeCoverageInstrumentation(); + + { + AutoCreatedBy acb(masm, "everything_else"); + Linker linker(masm); + if (masm.oom()) { + ReportOutOfMemory(cx); + return false; + } + + JitCode* code = linker.newCode(cx, CodeKind::Other); + if (!code) { + return false; + } + + // Register BaselineInterpreter code with the profiler's JitCode table. + { + auto entry = MakeJitcodeGlobalEntry( + cx, code, code->raw(), code->rawEnd()); + if (!entry) { + return false; + } + + JitcodeGlobalTable* globalTable = + cx->runtime()->jitRuntime()->getJitcodeGlobalTable(); + if (!globalTable->addEntry(std::move(entry))) { + ReportOutOfMemory(cx); + return false; + } + + code->setHasBytecodeMap(); + } + + // Patch loads now that we know the tableswitch base address. + CodeLocationLabel tableLoc(code, CodeOffset(tableOffset_)); + for (CodeOffset off : tableLabels_) { + MacroAssembler::patchNearAddressMove(CodeLocationLabel(code, off), + tableLoc); + } + + perfSpewer_.saveProfile(code); + +#ifdef MOZ_VTUNE + vtune::MarkStub(code, "BaselineInterpreter"); +#endif + + interpreter.init( + code, interpretOpOffset_, interpretOpNoDebugTrapOffset_, + bailoutPrologueOffset_.offset(), + profilerEnterFrameToggleOffset_.offset(), + profilerExitFrameToggleOffset_.offset(), debugTrapHandlerOffset_, + std::move(handler.debugInstrumentationOffsets()), + std::move(debugTrapOffsets_), std::move(handler.codeCoverageOffsets()), + std::move(handler.icReturnOffsets()), handler.callVMOffsets()); + } + + if (cx->runtime()->geckoProfiler().enabled()) { + interpreter.toggleProfilerInstrumentation(true); + } + + if (coverage::IsLCovEnabled()) { + interpreter.toggleCodeCoverageInstrumentationUnchecked(true); + } + + return true; +} + +JitCode* JitRuntime::generateDebugTrapHandler(JSContext* cx, + DebugTrapHandlerKind kind) { + TempAllocator temp(&cx->tempLifoAlloc()); + StackMacroAssembler masm(cx, temp); + AutoCreatedBy acb(masm, "JitRuntime::generateDebugTrapHandler"); + + AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All()); + MOZ_ASSERT(!regs.has(FramePointer)); + regs.takeUnchecked(ICStubReg); + if (HasInterpreterPCReg()) { + regs.takeUnchecked(InterpreterPCReg); + } +#ifdef JS_CODEGEN_ARM + regs.takeUnchecked(BaselineSecondScratchReg); + masm.setSecondScratchReg(BaselineSecondScratchReg); +#endif + Register scratch1 = regs.takeAny(); + Register scratch2 = regs.takeAny(); + Register scratch3 = regs.takeAny(); + + if (kind == DebugTrapHandlerKind::Interpreter) { + // The interpreter calls this for every script when debugging, so check if + // the script has any breakpoints or is in step mode before calling into + // C++. + Label hasDebugScript; + Address scriptAddr(FramePointer, + BaselineFrame::reverseOffsetOfInterpreterScript()); + masm.loadPtr(scriptAddr, scratch1); + masm.branchTest32(Assembler::NonZero, + Address(scratch1, JSScript::offsetOfMutableFlags()), + Imm32(int32_t(JSScript::MutableFlags::HasDebugScript)), + &hasDebugScript); + masm.abiret(); + masm.bind(&hasDebugScript); + + if (HasInterpreterPCReg()) { + // Update frame's bytecode pc because the debugger depends on it. + Address pcAddr(FramePointer, + BaselineFrame::reverseOffsetOfInterpreterPC()); + masm.storePtr(InterpreterPCReg, pcAddr); + } + } + + // Load the return address in scratch1. + masm.loadAbiReturnAddress(scratch1); + + // Load BaselineFrame pointer in scratch2. + masm.loadBaselineFramePtr(FramePointer, scratch2); + + // Enter a stub frame and call the HandleDebugTrap VM function. Ensure + // the stub frame has a nullptr ICStub pointer, since this pointer is marked + // during GC. + masm.movePtr(ImmPtr(nullptr), ICStubReg); + EmitBaselineEnterStubFrame(masm, scratch3); + + using Fn = bool (*)(JSContext*, BaselineFrame*, const uint8_t*); + VMFunctionId id = VMFunctionToId::id; + TrampolinePtr code = cx->runtime()->jitRuntime()->getVMWrapper(id); + + masm.push(scratch1); + masm.push(scratch2); + EmitBaselineCallVM(code, masm); + + EmitBaselineLeaveStubFrame(masm); + + if (kind == DebugTrapHandlerKind::Interpreter) { + // We have to reload the bytecode pc register. + Address pcAddr(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC()); + masm.loadPtr(pcAddr, InterpreterPCRegAtDispatch); + } + masm.abiret(); + + Linker linker(masm); + JitCode* handlerCode = linker.newCode(cx, CodeKind::Other); + if (!handlerCode) { + return nullptr; + } + + CollectPerfSpewerJitCodeProfile(handlerCode, "DebugTrapHandler"); + +#ifdef MOZ_VTUNE + vtune::MarkStub(handlerCode, "DebugTrapHandler"); +#endif + + return handlerCode; +} + +} // namespace jit +} // namespace js diff --git a/js/src/jit/BaselineCodeGen.h b/js/src/jit/BaselineCodeGen.h new file mode 100644 index 0000000000..16df7d3957 --- /dev/null +++ b/js/src/jit/BaselineCodeGen.h @@ -0,0 +1,521 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * vim: set ts=8 sts=2 et sw=2 tw=80: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef jit_BaselineCodeGen_h +#define jit_BaselineCodeGen_h + +#include "jit/BaselineFrameInfo.h" +#include "jit/BytecodeAnalysis.h" +#include "jit/FixedList.h" +#include "jit/MacroAssembler.h" +#include "jit/PerfSpewer.h" + +namespace js { + +namespace jit { + +enum class ScriptGCThingType { + Atom, + String, + RegExp, + Object, + Function, + Scope, + BigInt +}; + +// Base class for BaselineCompiler and BaselineInterpreterGenerator. The Handler +// template is a class storing fields/methods that are interpreter or compiler +// specific. This can be combined with template specialization of methods in +// this class to specialize behavior. +template +class BaselineCodeGen { + protected: + Handler handler; + + JSContext* cx; + StackMacroAssembler masm; + + typename Handler::FrameInfoT& frame; + + // Shared epilogue code to return to the caller. + NonAssertingLabel return_; + + NonAssertingLabel postBarrierSlot_; + + // Prologue code where we resume for Ion prologue bailouts. + NonAssertingLabel bailoutPrologue_; + + CodeOffset profilerEnterFrameToggleOffset_; + CodeOffset profilerExitFrameToggleOffset_; + + // Early Ion bailouts will enter at this address. This is after frame + // construction and before environment chain is initialized. + CodeOffset bailoutPrologueOffset_; + + // Baseline Interpreter can enter Baseline Compiler code at this address. This + // is right after the warm-up counter check in the prologue. + CodeOffset warmUpCheckPrologueOffset_; + + uint32_t pushedBeforeCall_ = 0; +#ifdef DEBUG + bool inCall_ = false; +#endif + + template + explicit BaselineCodeGen(JSContext* cx, TempAllocator& alloc, + HandlerArgs&&... args); + + template + void pushArg(const T& t) { + masm.Push(t); + } + + // Pushes the current script as argument for a VM function. + void pushScriptArg(); + + // Pushes the bytecode pc as argument for a VM function. + void pushBytecodePCArg(); + + // Pushes a name/object/scope associated with the current bytecode op (and + // stored in the script) as argument for a VM function. + void loadScriptGCThing(ScriptGCThingType type, Register dest, + Register scratch); + void pushScriptGCThingArg(ScriptGCThingType type, Register scratch1, + Register scratch2); + void pushScriptNameArg(Register scratch1, Register scratch2); + + // Pushes a bytecode operand as argument for a VM function. + void pushUint8BytecodeOperandArg(Register scratch); + void pushUint16BytecodeOperandArg(Register scratch); + + void loadInt32LengthBytecodeOperand(Register dest); + void loadNumFormalArguments(Register dest); + + // Loads the current JSScript* in dest. + void loadScript(Register dest); + + void saveInterpreterPCReg(); + void restoreInterpreterPCReg(); + + // Subtracts |script->nslots() * sizeof(Value)| from reg. + void subtractScriptSlotsSize(Register reg, Register scratch); + + // Jump to the script's resume entry indicated by resumeIndex. + void jumpToResumeEntry(Register resumeIndex, Register scratch1, + Register scratch2); + + // Load the global's lexical environment. + void loadGlobalLexicalEnvironment(Register dest); + void pushGlobalLexicalEnvironmentValue(ValueOperand scratch); + + // Load the |this|-value from the global's lexical environment. + void loadGlobalThisValue(ValueOperand dest); + + // Computes the frame size. See BaselineFrame::debugFrameSize_. + void computeFrameSize(Register dest); + + void prepareVMCall(); + + void storeFrameSizeAndPushDescriptor(uint32_t argSize, Register scratch); + + enum class CallVMPhase { BeforePushingLocals, AfterPushingLocals }; + bool callVMInternal(VMFunctionId id, RetAddrEntry::Kind kind, + CallVMPhase phase); + + template + bool callVM(RetAddrEntry::Kind kind = RetAddrEntry::Kind::CallVM, + CallVMPhase phase = CallVMPhase::AfterPushingLocals); + + template + bool callVMNonOp(CallVMPhase phase = CallVMPhase::AfterPushingLocals) { + return callVM(RetAddrEntry::Kind::NonOpCallVM, phase); + } + + // ifDebuggee should be a function emitting code for when the script is a + // debuggee script. ifNotDebuggee (if present) is called to emit code for + // non-debuggee scripts. + template + [[nodiscard]] bool emitDebugInstrumentation( + const F1& ifDebuggee, const mozilla::Maybe& ifNotDebuggee); + template + [[nodiscard]] bool emitDebugInstrumentation(const F& ifDebuggee) { + return emitDebugInstrumentation(ifDebuggee, mozilla::Maybe()); + } + + bool emitSuspend(JSOp op); + + template + [[nodiscard]] bool emitAfterYieldDebugInstrumentation(const F& ifDebuggee, + Register scratch); + + // ifSet should be a function emitting code for when the script has |flag| + // set. ifNotSet emits code for when the flag isn't set. + template + [[nodiscard]] bool emitTestScriptFlag(JSScript::ImmutableFlags flag, + const F1& ifSet, const F2& ifNotSet, + Register scratch); + + // If |script->hasFlag(flag) == value|, execute the code emitted by |emit|. + template + [[nodiscard]] bool emitTestScriptFlag(JSScript::ImmutableFlags flag, + bool value, const F& emit, + Register scratch); + template + [[nodiscard]] bool emitTestScriptFlag(JSScript::MutableFlags flag, bool value, + const F& emit, Register scratch); + + [[nodiscard]] bool emitEnterGeneratorCode(Register script, + Register resumeIndex, + Register scratch); + + void emitInterpJumpToResumeEntry(Register script, Register resumeIndex, + Register scratch); + void emitJumpToInterpretOpLabel(); + + [[nodiscard]] bool emitCheckThis(ValueOperand val, bool reinit = false); + void emitLoadReturnValue(ValueOperand val); + void emitGetAliasedVar(ValueOperand dest); + [[nodiscard]] bool emitGetAliasedDebugVar(ValueOperand dest); + + [[nodiscard]] bool emitNextIC(); + [[nodiscard]] bool emitInterruptCheck(); + [[nodiscard]] bool emitWarmUpCounterIncrement(); + +#define EMIT_OP(op, ...) bool emit_##op(); + FOR_EACH_OPCODE(EMIT_OP) +#undef EMIT_OP + + // JSOp::Pos, JSOp::Neg, JSOp::BitNot, JSOp::Inc, JSOp::Dec, JSOp::ToNumeric. + [[nodiscard]] bool emitUnaryArith(); + + // JSOp::BitXor, JSOp::Lsh, JSOp::Add etc. + [[nodiscard]] bool emitBinaryArith(); + + // Handles JSOp::Lt, JSOp::Gt, and friends + [[nodiscard]] bool emitCompare(); + + // Handles JSOp::NewObject and JSOp::NewInit. + [[nodiscard]] bool emitNewObject(); + + // For a JOF_JUMP op, jumps to the op's jump target. + void emitJump(); + + // For a JOF_JUMP op, jumps to the op's jump target depending on the Value + // in |val|. + void emitTestBooleanTruthy(bool branchIfTrue, ValueOperand val); + + // Converts |val| to an index in the jump table and stores this in |dest| + // or branches to the default pc if not int32 or out-of-range. + void emitGetTableSwitchIndex(ValueOperand val, Register dest, + Register scratch1, Register scratch2); + + // Jumps to the target of a table switch based on |key| and the + // firstResumeIndex stored in JSOp::TableSwitch. + void emitTableSwitchJump(Register key, Register scratch1, Register scratch2); + + [[nodiscard]] bool emitReturn(); + + [[nodiscard]] bool emitTest(bool branchIfTrue); + [[nodiscard]] bool emitAndOr(bool branchIfTrue); + [[nodiscard]] bool emitCoalesce(); + + [[nodiscard]] bool emitCall(JSOp op); + [[nodiscard]] bool emitSpreadCall(JSOp op); + + [[nodiscard]] bool emitDelElem(bool strict); + [[nodiscard]] bool emitDelProp(bool strict); + [[nodiscard]] bool emitSetElemSuper(bool strict); + [[nodiscard]] bool emitSetPropSuper(bool strict); + + // Try to bake in the result of BindGName instead of using an IC. + // Return true if we managed to optimize the op. + bool tryOptimizeBindGlobalName(); + + [[nodiscard]] bool emitInitPropGetterSetter(); + [[nodiscard]] bool emitInitElemGetterSetter(); + + [[nodiscard]] bool emitFormalArgAccess(JSOp op); + + [[nodiscard]] bool emitUninitializedLexicalCheck(const ValueOperand& val); + + [[nodiscard]] bool emitIsMagicValue(); + + void getEnvironmentCoordinateObject(Register reg); + Address getEnvironmentCoordinateAddressFromObject(Register objReg, + Register reg); + Address getEnvironmentCoordinateAddress(Register reg); + + [[nodiscard]] bool emitPrologue(); + [[nodiscard]] bool emitEpilogue(); + [[nodiscard]] bool emitOutOfLinePostBarrierSlot(); + [[nodiscard]] bool emitStackCheck(); + [[nodiscard]] bool emitDebugPrologue(); + [[nodiscard]] bool emitDebugEpilogue(); + + [[nodiscard]] bool initEnvironmentChain(); + + [[nodiscard]] bool emitHandleCodeCoverageAtPrologue(); + + void emitInitFrameFields(Register nonFunctionEnv); + [[nodiscard]] bool emitIsDebuggeeCheck(); + void emitInitializeLocals(); + + void emitProfilerEnterFrame(); + void emitProfilerExitFrame(); +}; + +using RetAddrEntryVector = js::Vector; + +// Interface used by BaselineCodeGen for BaselineCompiler. +class BaselineCompilerHandler { + CompilerFrameInfo frame_; + TempAllocator& alloc_; + BytecodeAnalysis analysis_; +#ifdef DEBUG + const MacroAssembler& masm_; +#endif + FixedList